use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.
the class ReadEventsTestParams method testMissingEOWUnhappyPathWithMultiEventWindow.
/**
* Testcase sends events with appropriate EOW events in between:
* 2 valid packets, no EOW for prior packet, 2 valid packets with a higher scn.
* @throws Exception
*/
@Test
public void testMissingEOWUnhappyPathWithMultiEventWindow() throws Exception {
final DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(1000, 1000, 100, 500, AllocationPolicy.HEAP_MEMORY, QueuePolicy.BLOCK_ON_WRITE, AssertLevel.NONE));
dbusBuf.setDropOldEvents(true);
DbusEventGenerator generator = new DbusEventGenerator();
Vector<DbusEvent> events = new Vector<DbusEvent>();
generator.generateEvents(6, 1, 100, 10, events);
// conversion of readable events to writable is for TESTING ONLY:
DbusEventInternalWritable writableEvent = DbusEventCorrupter.makeWritable(events.get(1));
writableEvent.setSequence(events.get(0).sequence());
writableEvent.applyCrc();
writableEvent = DbusEventCorrupter.makeWritable(events.get(2));
writableEvent.setSequence(events.get(0).sequence());
writableEvent.applyCrc();
writableEvent = DbusEventCorrupter.makeWritable(events.get(3));
// NO EOW for event 0 sequence.
writableEvent.setSequence(events.get(0).sequence() + 100);
writableEvent.applyCrc();
writableEvent = DbusEventCorrupter.makeWritable(events.get(4));
writableEvent.setSequence(events.get(3).sequence());
writableEvent.applyCrc();
writableEvent = DbusEventCorrupter.makeWritable(events.get(5));
writableEvent.setSrcId((short) -2);
writableEvent.applyCrc();
// Increment the SCN and reuse
for (int i = 0; i < 6; ++i) {
DbusEvent e = events.get(i);
assertTrue("invalid event #" + i, e.isValid());
}
// Set up the ReadChannel with 6 events
ByteArrayOutputStream oStream = new ByteArrayOutputStream();
WritableByteChannel oChannel = Channels.newChannel(oStream);
for (int i = 0; i < 6; ++i) {
((DbusEventInternalReadable) events.get(i)).writeTo(oChannel, Encoding.BINARY);
}
byte[] writeBytes = oStream.toByteArray();
ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
final ReadableByteChannel rChannel = Channels.newChannel(iStream);
try {
dbusBuf.readEvents(rChannel);
Assert.fail();
} catch (InvalidEventException ie) {
LOG.info("Exception trace is " + ie);
return;
}
}
use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.
the class ReadEventsTestParams method testStreamScn.
@Test
public void testStreamScn() throws Exception {
// Src Event producer
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
EventBufferTestInput input = new EventBufferTestInput();
int numEvents = 500;
int numScns = 10;
int windowSize = numEvents / numScns;
input.setNumEvents(numEvents).setWindowSize(windowSize).setSharedBufferSize(numEvents * 2).setStagingBufferSize(numEvents * 2).setIndexSize(numEvents / 10).setIndividualBufferSize(numEvents * 2).setBatchSize(numEvents * 2).setProducerBufferSize(numEvents * 2).setPayloadSize(100).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
input.setTestName("testStreamScn");
DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(1, "clientStats", true, true, null);
DbusEventGenerator evGen = new DbusEventGenerator();
assertTrue(evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), true, srcTestEvents) > 0);
int eventSize = srcTestEvents.get(0).size();
DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
DbusEventBuffer readEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats);
Vector<Long> seenScns = new Vector<Long>();
Checkpoint cp = new Checkpoint();
cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
boolean origEmptyValue = prodEventBuffer.empty();
// empty buffer; prevScn=-1 , minScn=-1 ; so no Scn not found exception
cp.setWindowScn(2L);
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
boolean res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(-1L, prodEventBuffer.getPrevScn());
assertTrue(res);
assertEquals(0, seenScns.size());
// partial buffer; with no complete window written; prevScn > sinceScn , minScn=-1 ; Scn not found exception thrown;
cp.setWindowScn(2L);
cp.setWindowOffset(-1);
prodEventBuffer.setPrevScn(20L);
prodEventBuffer.setEmpty(false);
seenScns.clear();
readEventBuffer.clear();
assertEquals(-1L, prodEventBuffer.getMinScn());
res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertFalse(res);
assertEquals(0, seenScns.size());
// restore
prodEventBuffer.setPrevScn(-1L);
prodEventBuffer.setEmpty(origEmptyValue);
// partial buffer; with no complete window written; sinceScn >= prevScn , minScn=-1 ; no exception;
cp.setWindowScn(45L);
cp.setWindowOffset(-1);
prodEventBuffer.setPrevScn(20L);
prodEventBuffer.setEmpty(false);
seenScns.clear();
readEventBuffer.clear();
res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertTrue(res);
assertEquals(0, seenScns.size());
// restore
prodEventBuffer.setPrevScn(-1L);
prodEventBuffer.setEmpty(origEmptyValue);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
tEmitter.join();
long minScn = emitterStats.getTotalStats().getMinScn();
long maxScn = emitterStats.getTotalStats().getMaxScn();
long prevScn = emitterStats.getTotalStats().getPrevScn();
System.out.printf("minScn=%d,maxScn=%d,prevScn=%d,range=%d\n", minScn, maxScn, prevScn, emitterStats.getTotalStats().getTimeSpan());
assertEquals(numEvents - 1, emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS);
assertEquals(prodEventBuffer.getTimestampOfFirstEvent(), emitterStats.getTotalStats().getTimestampMinScnEvent());
// stream with scn < max; expect last window; not last 2
cp.setWindowScn(maxScn - 1);
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(1, seenScns.size());
assertEquals(Long.valueOf(maxScn), seenScns.get(0));
// set windowScn to maxScn; get >= behaviour here ; get the last window
cp.setWindowScn(maxScn);
cp.setWindowOffset(0);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(1, seenScns.size());
assertEquals(Long.valueOf(maxScn), seenScns.get(0));
// stream with scn >= max ; get a window higher than max - expect nothing
cp.setWindowScn(maxScn);
cp.setWindowOffset(-1);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(0, seenScns.size());
// stream with scn > max
cp.setWindowScn(maxScn + 1);
cp.setWindowOffset(-1);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(0, seenScns.size());
// stream with scn >= min
cp.setWindowScn(minScn);
cp.setWindowOffset(-1);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(numScns - 1, seenScns.size());
assertTrue(seenScns.get(0) != minScn);
assertEquals(Long.valueOf(maxScn), seenScns.get(numScns - 2));
// stream with scn < min but >= prevScn
cp.setWindowScn(prevScn);
cp.setWindowOffset(-1);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns, clientStats);
assertEquals(numScns, seenScns.size());
assertEquals(Long.valueOf(minScn), seenScns.get(0));
System.out.printf("Clientstats: minScn=%d , maxScn=%d , timespan=%d timeSinceLastEvent=%d\n", clientStats.getTotalStats().getMinScn(), clientStats.getTotalStats().getMaxScn(), clientStats.getTotalStats().getTimeSpan(), clientStats.getTotalStats().getTimeSinceLastEvent());
assertEquals(maxScn, clientStats.getTotalStats().getMaxScn());
assertEquals(numEvents - 1, clientStats.getTotalStats().getTimeSpan() / MILLISECONDS);
assertEquals(clientStats.getTotalStats().getTimestampMaxScnEvent(), emitterStats.getTotalStats().getTimestampMaxScnEvent());
assertEquals(minScn, clientStats.getTotalStats().getMinScn());
// stream with scn < prevScn
cp.setWindowScn(prevScn - 1);
cp.setWindowOffset(-1);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(0, seenScns.size());
// stream with scn == prevScn but windowOffset=0
cp.setWindowScn(prevScn);
cp.setWindowOffset(0);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(0, seenScns.size());
// stream with scn > min
cp.setWindowScn(minScn + 1);
cp.setWindowOffset(-1);
readEventBuffer.clear();
seenScns.clear();
streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(numScns - 1, seenScns.size());
assertEquals(Long.valueOf(maxScn), seenScns.get(numScns - 2));
assertTrue(seenScns.get(0) != minScn);
}
use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.
the class ReadEventsTestParams method testStatsMinMaxScn.
@Test
public void testStatsMinMaxScn() throws Exception {
// Src Event producer
Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
EventBufferTestInput input = new EventBufferTestInput();
int numEvents = 10000;
long startScn = 1000;
int numWinScn = 10;
// set sharedBufferSize to a value much smaller than total size required
input.setNumEvents(numEvents).setWindowSize(numEvents / numWinScn).setSharedBufferSize(numEvents / 5).setStagingBufferSize(numEvents / 5).setIndexSize(numEvents / 10).setIndividualBufferSize(numEvents / 2).setBatchSize(numEvents / 5).setProducerBufferSize(numEvents / 2).setPayloadSize(100).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
input.setTestName("testStatsMinMaxScn");
DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
DbusEventGenerator evGen = new DbusEventGenerator(startScn);
if (evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), true, srcTestEvents) <= 0) {
fail();
return;
}
// sleep 10 ms;
int eventSize = srcTestEvents.get(0).size();
DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats);
Thread tEmitter = new Thread(eventProducer);
tEmitter.start();
tEmitter.join();
// sleep 10 ms;
int msDelay = 10;
Thread.sleep(msDelay);
long min = (numWinScn - 3) * input.getWindowSize() + startScn;
long max = numWinScn * input.getWindowSize() + startScn;
// note : event generator generates events such that a one second lag exists between the latest event and prev event
long expectedRange = (max - min) + input.getWindowSize() - 1;
System.out.printf("Total timespan = %d\n", (srcTestEvents.get(numEvents - 1).timestampInNanos() - srcTestEvents.get(0).timestampInNanos()) / NANOSECONDS);
System.out.printf("prevScn=%d\n", emitterStats.getTotalStats().getPrevScn());
System.out.printf("min = %d , max=%d buf=%d ,%d\n", emitterStats.getTotalStats().getMinScn(), emitterStats.getTotalStats().getMaxScn(), prodEventBuffer.getMinScn(), prodEventBuffer.lastWrittenScn());
System.out.printf("timespan=%d , timeSinceLastEvent = %d , timeSinceLastAccess %d\n", emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS, emitterStats.getTotalStats().getTimeSinceLastEvent(), emitterStats.getTotalStats().getTimeSinceLastAccess());
assertEquals(numEvents, srcTestEvents.size());
assertEquals(numEvents, emitterStats.getTotalStats().getNumDataEvents());
assertEquals(min, emitterStats.getTotalStats().getMinScn());
assertEquals(max, emitterStats.getTotalStats().getMaxScn());
assertEquals(min - input.getWindowSize(), emitterStats.getTotalStats().getPrevScn());
assertEquals(emitterStats.getTotalStats().getSizeDataEvents() * numEvents, numEvents * eventSize);
long tsSpanInSec = emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS;
assertEquals(expectedRange, tsSpanInSec);
long tsSinceLastEvent = emitterStats.getTotalStats().getTimeSinceLastEvent();
assertTrue(tsSinceLastEvent >= msDelay);
assertTrue(emitterStats.getTotalStats().getTimeLag() >= 0);
assertTrue(emitterStats.getTotalStats().getMinTimeLag() >= 0);
assertTrue(emitterStats.getTotalStats().getMaxTimeLag() >= 0);
DbusEventBuffer readEventBuffer = new DbusEventBuffer(getConfig(numEvents * 2L * eventSize, numEvents * eventSize, (numEvents / 10) * eventSize, numEvents * 2 * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
// check streaming
Vector<Long> seenScns = new Vector<Long>();
Checkpoint cp = new Checkpoint();
cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
// case: where sinceScn < min , > prevScn ; return the entire buffer
cp.setWindowScn(min - 1);
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
int expectedNumWin = (int) (max - min) / input.getWindowSize() + 1;
assertEquals(expectedNumWin, seenScns.size());
assertEquals(Long.valueOf(min), seenScns.get(0));
assertEquals(Long.valueOf(max), seenScns.get(expectedNumWin - 1));
// case : where sinceScn < prevScn ; exception thrown;
cp.setWindowScn(startScn);
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(0, seenScns.size());
// case: where sinceScn < min , = prevScn ; offset=-1 ,return the entire buffer
cp.setWindowScn(prodEventBuffer.getPrevScn());
cp.setWindowOffset(-1);
seenScns.clear();
readEventBuffer.clear();
streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
assertEquals(expectedNumWin, seenScns.size());
assertEquals(Long.valueOf(min), seenScns.get(0));
assertEquals(Long.valueOf(max), seenScns.get(expectedNumWin - 1));
// case: where sinceScn < min , = prevScn ; offset=0 ,return nothing
cp.setWindowScn(prodEventBuffer.getPrevScn());
cp.setWindowOffset(0);
seenScns.clear();
readEventBuffer.clear();
assertFalse(streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns));
assertEquals(0, seenScns.size());
}
use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.
the class ReadEventsTestParams method testReadEventsAssertSpanError.
/**
* Test to reproduce DDSDBUS-388.
*/
@Test
public void testReadEventsAssertSpanError() throws Exception {
final Logger log = Logger.getLogger("TestDbusEventBuffer.testReadEventsAssertSpanError");
// log.setLevel(Level.INFO);
log.info("start");
DbusEventBuffer dbusBuf = new DbusEventBuffer(getConfig(10000, 10000, 320, 5000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
// dbusBuf.getLog().setLevel(Level.DEBUG);
DbusEventGenerator generator = new DbusEventGenerator();
Vector<DbusEvent> events = new Vector<DbusEvent>();
// event size: 10 + 61 = 71
// window size: 5 * 71 + 61 = 416
// layout: [61 + 23 * 416 + 5 * 71 = 9984 ] <WrapAround> [ 61 + 22 * 416 + 2 * 71 + 61 = 9416 ]
// [ H=Gen:0,Ofs:0, T=Gen:0,Ofs:9984, L=10000 ]
// [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9416, L=9984 ]
generator.generateEvents(232, 5, 100, 10, events);
// Add events to the EventBuffer
DbusEventAppender appender = new DbusEventAppender(events, dbusBuf, null);
appender.run();
log.info("Dbus Event Buffer is :" + dbusBuf);
log.info("SCNIndex is :" + dbusBuf.getScnIndex());
final BufferPosition headPos = new BufferPosition(dbusBuf.getHead(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
final BufferPosition tailPos = new BufferPosition(dbusBuf.getTail(), dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
Assert.assertEquals(0, headPos.bufferGenId());
Assert.assertEquals(0, headPos.bufferIndex());
Assert.assertEquals(9629, headPos.bufferOffset());
Assert.assertEquals(1, tailPos.bufferGenId());
Assert.assertEquals(0, tailPos.bufferIndex());
Assert.assertEquals(9416, tailPos.bufferOffset());
Assert.assertEquals(9984, dbusBuf.getBuffer()[0].limit());
dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
final BufferPosition lastWinStart = new BufferPosition(61 + 22 * 416, dbusBuf.getBufferPositionParser(), dbusBuf.getBuffer());
dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
long lastScn = events.get(events.size() - 1).sequence();
generator = new DbusEventGenerator(lastScn - 1);
events.clear();
// event size: 10 + 61 = 71
generator.generateEvents(2, 2, 75, 10, events);
// intended layout: append 2 * 71 = 142 bytes
// [ H=Gen:0,Ofs:9629, T=Gen:1,Ofs:9558, L=9984 ]
lastScn = events.get(events.size() - 1).sequence();
generator = new DbusEventGenerator(lastScn - 1);
// event size: 500 + 61 = 561
// partial window size: 561 = 622
generator.generateEvents(1, 1, 800, 500, events);
// intended layout: append 561 bytes;
// [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
// head moves to first window after 561: 61 + 2 * 416 = 893 > 561
// Set up the ReadChannel with new events
ByteArrayOutputStream oStream = new ByteArrayOutputStream();
WritableByteChannel oChannel = Channels.newChannel(oStream);
for (DbusEvent e : events) {
assertTrue("invalid event", e.isValid());
((DbusEventInternalReadable) e).writeTo(oChannel, Encoding.BINARY);
log.debug("event size is: " + e.size());
}
byte[] writeBytes = oStream.toByteArray();
ByteArrayInputStream iStream = new ByteArrayInputStream(writeBytes);
final ReadableByteChannel rChannel = Channels.newChannel(iStream);
dbusBuf.readEvents(rChannel);
log.info("Dbus Event Buffer is :" + dbusBuf);
log.info("SCNIndex is :" + dbusBuf.getScnIndex());
// expected layout:
// [ H=Gen:1,Ofs:893, T=Gen:2,Ofs:561, L=9558 ]
headPos.setPosition(dbusBuf.getHead());
Assert.assertEquals(1, headPos.bufferGenId());
Assert.assertEquals(0, headPos.bufferIndex());
Assert.assertEquals(893, headPos.bufferOffset());
tailPos.setPosition(dbusBuf.getTail());
Assert.assertEquals(2, tailPos.bufferGenId());
Assert.assertEquals(0, tailPos.bufferIndex());
Assert.assertEquals(561, tailPos.bufferOffset());
Assert.assertEquals(9558, dbusBuf.getBuffer()[0].limit());
dbusBuf.getBufferPositionParser().assertSpan(dbusBuf.getHead(), dbusBuf.getTail(), true);
dbusBuf.getScnIndex().assertHeadPosition(dbusBuf.getHead());
// the second write did not add EOP so the SCN index tail should not have changed
dbusBuf.getScnIndex().assertLastWrittenPos(lastWinStart);
}
use of com.linkedin.databus.core.test.DbusEventGenerator in project databus by linkedin.
the class ReadEventsTestParams method generateEvents.
public void generateEvents() {
_log.info("Generating events");
_evGen = new DbusEventGenerator(_startScn);
_srcEvents = new Vector<DbusEvent>(_numSrcEvents);
Assert.assertTrue(_evGen.generateEvents(_numSrcEvents, _maxWindowSize, _eventSize + 300, _eventSize, _srcEvents) > 0);
}
Aggregations