Examples with DbusEventsStatisticsCollector com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector used on opensource projects

Example 36 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class TestDbusEventBufferMult method testSubscriptionStream.

@Test
public void testSubscriptionStream() throws Exception {
    final Logger log = Logger.getLogger("TestDbusEventBufferMult.testSubscriptionStream");
    log.info("start");
    TestSetup t = new TestSetup();
    PhysicalPartition pp100 = new PhysicalPartition(100, "multBufferTest1");
    PhysicalPartitionKey pk1 = new PhysicalPartitionKey(pp100);
    PhysicalPartition pp101 = new PhysicalPartition(101, "multBufferTest2");
    PhysicalPartitionKey pk2 = new PhysicalPartitionKey(pp101);
    //generate events in pp100
    byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
    DbusEventBufferAppendable buf100 = t._eventBuffer.getDbusEventBufferAppendable(pp100);
    buf100.startEvents();
    assertTrue(buf100.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 1, schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(10), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 1, schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(11), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 1, schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(2), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    buf100.endEvents(100, null);
    buf100.startEvents();
    assertTrue(buf100.appendEvent(new DbusEventKey(3), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    assertTrue(buf100.appendEvent(new DbusEventKey(4), (short) 100, (short) 1, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    buf100.endEvents(200, null);
    //generate events in pp100
    DbusEventBufferAppendable buf101 = t._eventBuffer.getDbusEventBufferAppendable(pp101);
    buf101.startEvents();
    assertTrue(buf101.appendEvent(new DbusEventKey(51), (short) 101, (short) 0, System.currentTimeMillis() * 1000000, (short) 11, schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(52), (short) 101, (short) 0, System.currentTimeMillis() * 1000000, (short) 12, schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(53), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    buf101.endEvents(120, null);
    buf101.startEvents();
    assertTrue(buf101.appendEvent(new DbusEventKey(54), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(55), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    assertTrue(buf101.appendEvent(new DbusEventKey(56), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
    buf101.endEvents(200, null);
    //initialization
    DatabusSubscription sub1 = DatabusSubscription.createPhysicalPartitionReplicationSubscription(new PhysicalPartition(100, "multBufferTest1"));
    DbusFilter filter1 = t._eventBuffer.constructFilters(Arrays.asList(sub1));
    assertNotNull(filter1);
    CheckpointMult cpMult1 = new CheckpointMult();
    Checkpoint cp100 = new Checkpoint();
    cp100.init();
    cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp100.setWindowScn(10L);
    cp100.setWindowOffset(-1);
    cpMult1.addCheckpoint(pp100, cp100);
    String[] pnames = { "multBufferTest1:100", "multBufferTest2:101" };
    StatsCollectors<DbusEventsStatisticsCollector> statsColls1 = createStats(pnames);
    DbusEventsStatisticsCollector statsCol1 = statsColls1.getStatsCollector("multBufferTest1:100");
    DbusEventsStatisticsCollector statsCol2 = statsColls1.getStatsCollector("multBufferTest2:101");
    //read an entire buffer
    DbusEventBufferBatchReadable reader1 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult1, Arrays.asList(pk1), statsColls1);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    // Try a call with 20 bytes of fetch size, we should see the event size in the first return with 0 events read.
    StreamEventsResult result = reader1.streamEvents(false, 20, Channels.newChannel(baos), Encoding.BINARY, filter1);
    assertEquals(0, result.getNumEventsStreamed());
    assertEquals(161, result.getSizeOfPendingEvent());
    result = reader1.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter1);
    int eventsRead = result.getNumEventsStreamed();
    //4 events + 1 eop + 2 events + 1 eop
    assertEquals(eventsRead, 8);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEvents(), 6);
    assertEquals(result.getSizeOfPendingEvent(), 0, "Size of pending event not zero");
    // Now that we have read all the events, we should not see a pending event even if we offer a small fetch size.
    result = reader1.streamEvents(false, 20, Channels.newChannel(baos), Encoding.BINARY, filter1);
    assertEquals(0, result.getNumEventsStreamed(), "There should be no more events in the buffer now");
    assertEquals(0, result.getSizeOfPendingEvent(), "We should not see pending event size since there are no events in buffer");
    baos.reset();
    statsCol1.reset();
    statsCol2.reset();
    //read from two buffers, filtering out one
    cpMult1 = new CheckpointMult();
    cp100.init();
    cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp100.setWindowScn(10L);
    cp100.setWindowOffset(-1);
    cpMult1.addCheckpoint(pp100, cp100);
    reader1 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult1, Arrays.asList(pk1, pk2), statsColls1);
    eventsRead = reader1.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter1).getNumEventsStreamed();
    //4 events + 1 eop + 1 eop from the other buffer + 2 events +
    assertEquals(eventsRead, 10);
    //1 eop + 1 eop from the other buffer
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEvents(), 6);
    baos.reset();
    statsCol1.reset();
    //read from one buffer and one source partition
    DatabusSubscription sub2 = new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE, new PhysicalPartition(101, "multBufferTest2"), new LogicalSourceId(new LogicalSource(2, "srcName2"), (short) 2));
    DbusFilter filter2 = t._eventBuffer.constructFilters(Arrays.asList(sub2));
    assertNotNull(filter2);
    CheckpointMult cpMult2 = new CheckpointMult();
    Checkpoint cp101 = new Checkpoint();
    cp101.init();
    cp101.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp101.setWindowScn(10L);
    cp101.setWindowOffset(-1);
    cpMult2.addCheckpoint(pp101, cp101);
    DbusEventBufferBatchReadable reader2 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult2, Arrays.asList(pk2), statsColls1);
    eventsRead = reader2.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter2).getNumEventsStreamed();
    //1 events + 1 eop + 3events + 1 eop
    assertEquals(eventsRead, 6);
    baos.reset();
    statsCol1.reset();
    statsCol2.reset();
    //read all partitions for a source
    DatabusSubscription sub3 = new DatabusSubscription(PhysicalSource.MASTER_PHISYCAL_SOURCE, PhysicalPartition.ANY_PHYSICAL_PARTITION, LogicalSourceId.createAllPartitionsWildcard(new LogicalSource(2, "srcName2")));
    DbusFilter filter3 = t._eventBuffer.constructFilters(Arrays.asList(sub3));
    assertNotNull(filter3);
    CheckpointMult cpMult3 = new CheckpointMult();
    cp100.init();
    cp100.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp100.setWindowScn(10L);
    cp100.setWindowOffset(-1);
    cpMult1.addCheckpoint(pp100, cp100);
    cp101.init();
    cp101.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cp101.setWindowScn(10L);
    cp101.setWindowOffset(-1);
    cpMult2.addCheckpoint(pp101, cp101);
    DbusEventBufferBatchReadable reader3 = t._eventBuffer.getDbusEventBufferBatchReadable(cpMult3, Arrays.asList(pk1, pk2), statsColls1);
    eventsRead = reader3.streamEvents(false, 1000000, Channels.newChannel(baos), Encoding.BINARY, filter3).getNumEventsStreamed();
    //1 events + 1 eop + 1 events + 1 eop + 2 events + 1 eop + 3 events + 1 eop
    assertEquals(eventsRead, 11);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumSysEvents(), 2);
    assertEquals(statsColls1.getStatsCollector("multBufferTest2:101").getTotalStats().getNumSysEvents(), 2);
    assertEquals(statsColls1.getStatsCollector("multBufferTest1:100").getTotalStats().getNumDataEventsFiltered(), 3);
    assertEquals(statsColls1.getStatsCollector("multBufferTest2:101").getTotalStats().getNumDataEventsFiltered(), 4);
    baos.reset();
    statsCol1.reset();
    statsCol2.reset();
    log.info("end");
}

Example 37 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsTestParams method testStreamScn.

@Test
public void testStreamScn() throws Exception {
    //Src Event producer
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    EventBufferTestInput input = new EventBufferTestInput();
    int numEvents = 500;
    int numScns = 10;
    int windowSize = numEvents / numScns;
    input.setNumEvents(numEvents).setWindowSize(windowSize).setSharedBufferSize(numEvents * 2).setStagingBufferSize(numEvents * 2).setIndexSize(numEvents / 10).setIndividualBufferSize(numEvents * 2).setBatchSize(numEvents * 2).setProducerBufferSize(numEvents * 2).setPayloadSize(100).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
    input.setTestName("testStreamScn");
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(1, "clientStats", true, true, null);
    DbusEventGenerator evGen = new DbusEventGenerator();
    assertTrue(evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), true, srcTestEvents) > 0);
    int eventSize = srcTestEvents.get(0).size();
    DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
    DbusEventBuffer readEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats);
    Vector<Long> seenScns = new Vector<Long>();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    boolean origEmptyValue = prodEventBuffer.empty();
    //empty buffer; prevScn=-1 , minScn=-1 ; so no Scn not found exception
    cp.setWindowScn(2L);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    boolean res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(-1L, prodEventBuffer.getPrevScn());
    assertTrue(res);
    assertEquals(0, seenScns.size());
    //partial buffer; with no complete window written;  prevScn > sinceScn , minScn=-1 ; Scn not found exception thrown;
    cp.setWindowScn(2L);
    cp.setWindowOffset(-1);
    prodEventBuffer.setPrevScn(20L);
    prodEventBuffer.setEmpty(false);
    seenScns.clear();
    readEventBuffer.clear();
    assertEquals(-1L, prodEventBuffer.getMinScn());
    res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertFalse(res);
    assertEquals(0, seenScns.size());
    //restore
    prodEventBuffer.setPrevScn(-1L);
    prodEventBuffer.setEmpty(origEmptyValue);
    //partial buffer; with no complete window written;  sinceScn >= prevScn , minScn=-1 ; no exception;
    cp.setWindowScn(45L);
    cp.setWindowOffset(-1);
    prodEventBuffer.setPrevScn(20L);
    prodEventBuffer.setEmpty(false);
    seenScns.clear();
    readEventBuffer.clear();
    res = streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertTrue(res);
    assertEquals(0, seenScns.size());
    //restore
    prodEventBuffer.setPrevScn(-1L);
    prodEventBuffer.setEmpty(origEmptyValue);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();
    long minScn = emitterStats.getTotalStats().getMinScn();
    long maxScn = emitterStats.getTotalStats().getMaxScn();
    long prevScn = emitterStats.getTotalStats().getPrevScn();
    System.out.printf("minScn=%d,maxScn=%d,prevScn=%d,range=%d\n", minScn, maxScn, prevScn, emitterStats.getTotalStats().getTimeSpan());
    assertEquals(numEvents - 1, emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS);
    assertEquals(prodEventBuffer.getTimestampOfFirstEvent(), emitterStats.getTotalStats().getTimestampMinScnEvent());
    //stream with scn < max; expect last window; not last 2
    cp.setWindowScn(maxScn - 1);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(0));
    //set windowScn to maxScn; get >= behaviour here ; get the last window
    cp.setWindowScn(maxScn);
    cp.setWindowOffset(0);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(0));
    //stream with scn >= max ; get a window higher than max - expect nothing
    cp.setWindowScn(maxScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    //stream with scn > max
    cp.setWindowScn(maxScn + 1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    //stream with scn >= min
    cp.setWindowScn(minScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(numScns - 1, seenScns.size());
    assertTrue(seenScns.get(0) != minScn);
    assertEquals(Long.valueOf(maxScn), seenScns.get(numScns - 2));
    //stream with scn < min but >= prevScn
    cp.setWindowScn(prevScn);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns, clientStats);
    assertEquals(numScns, seenScns.size());
    assertEquals(Long.valueOf(minScn), seenScns.get(0));
    System.out.printf("Clientstats: minScn=%d , maxScn=%d , timespan=%d timeSinceLastEvent=%d\n", clientStats.getTotalStats().getMinScn(), clientStats.getTotalStats().getMaxScn(), clientStats.getTotalStats().getTimeSpan(), clientStats.getTotalStats().getTimeSinceLastEvent());
    assertEquals(maxScn, clientStats.getTotalStats().getMaxScn());
    assertEquals(numEvents - 1, clientStats.getTotalStats().getTimeSpan() / MILLISECONDS);
    assertEquals(clientStats.getTotalStats().getTimestampMaxScnEvent(), emitterStats.getTotalStats().getTimestampMaxScnEvent());
    assertEquals(minScn, clientStats.getTotalStats().getMinScn());
    //stream with scn < prevScn
    cp.setWindowScn(prevScn - 1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    //stream with scn == prevScn but windowOffset=0
    cp.setWindowScn(prevScn);
    cp.setWindowOffset(0);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    //stream with scn > min
    cp.setWindowScn(minScn + 1);
    cp.setWindowOffset(-1);
    readEventBuffer.clear();
    seenScns.clear();
    streamWriterReader(prodEventBuffer, input.getBatchSize() * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(numScns - 1, seenScns.size());
    assertEquals(Long.valueOf(maxScn), seenScns.get(numScns - 2));
    assertTrue(seenScns.get(0) != minScn);
}

Example 38 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsTestParams method testAppendEventStats.

@Test
public void testAppendEventStats() throws Exception {
    DbusEventsStatisticsCollector collector = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
    DbusEventBuffer dbuf = new DbusEventBuffer(getConfig(10000000, DbusEventBuffer.Config.DEFAULT_INDIVIDUAL_BUFFER_SIZE, 100000, 1000000, AllocationPolicy.HEAP_MEMORY, QueuePolicy.OVERWRITE_ON_WRITE, AssertLevel.ALL));
    // DDSDBUS-1109
    assertTrue(dbuf.getScnIndex().getUpdateOnNext());
    dbuf.startEvents();
    long now = System.currentTimeMillis();
    final int sleepTime = 100;
    Thread.sleep(sleepTime);
    assertTrue(dbuf.appendEvent(new DbusEventKey(key), pPartitionId, lPartitionId, now * 1000000, srcId, schemaId, value.getBytes(Charset.defaultCharset()), false));
    dbuf.endEvents(true, 0x100000001L, false, false, collector);
    assertTrue(collector.getTotalStats().getTimeLag() + "," + sleepTime, collector.getTotalStats().getTimeLag() >= sleepTime);
    assertTrue(collector.getTotalStats().getMinTimeLag() + "," + sleepTime, collector.getTotalStats().getMinTimeLag() >= sleepTime);
    assertTrue(collector.getTotalStats().getMaxTimeLag() + "," + sleepTime, collector.getTotalStats().getMaxTimeLag() >= sleepTime);
}

Example 39 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsTestParams method runReaderWriterTest.

// "enableAsserts" is an overstatement; the boolean merely checks the most
// common assertion failure and, if it's going to fail, bypasses that and
// all subsequent asserts.  It's still possible for the overall number of
// (data) events to check out but the number of system events to fail.
private boolean runReaderWriterTest(EventBufferTestInput testInput, boolean enableAsserts) throws Exception {
    // Src Event producer
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    // Dest Event consumer
    Vector<DbusEvent> dstTestEvents = new Vector<DbusEvent>();
    // emitter/producer/appender -> DEB -> writer -> pipe -> reader -> DEB -> consumer
    // Test configurations;
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
    DbusEventsStatisticsCollector streamStats = new DbusEventsStatisticsCollector(1, "streamStats", true, true, null);
    DbusEventsStatisticsCollector clientStats = new DbusEventsStatisticsCollector(1, "clientStats", true, true, null);
    srcTestEvents.clear();
    dstTestEvents.clear();
    // a.k.a. producer, a.k.a. appender
    emitterStats.reset();
    // a.k.a. writer (== reader from relay buffer)
    streamStats.reset();
    // a.k.a. reader (== writer into client buffer)
    clientStats.reset();
    assertEquals(0, dstTestEvents.size());
    assertTrue(runConstEventsReaderWriter(srcTestEvents, dstTestEvents, testInput, emitterStats, streamStats, clientStats));
    int numEvents = testInput.getNumEvents();
    // check data!
    assertEquals("numEvents generated", numEvents, srcTestEvents.size());
    if (!enableAsserts && numEvents != dstTestEvents.size()) {
        LOG.info(testInput.getTestName() + " would fail if asserts were enabled:  numEvents seen by consumer = " + dstTestEvents.size() + " < " + numEvents);
        return false;
    }
    assertEquals(testInput.getTestName() + ": numEvents seen by consumer", numEvents, dstTestEvents.size());
    assertEquals(testInput.getTestName() + ": numDataEvents seen by producer", srcTestEvents.size(), emitterStats.getTotalStats().getNumDataEvents());
    assertEquals(testInput.getTestName() + ": numDataEvents seen by writer", dstTestEvents.size(), streamStats.getTotalStats().getNumDataEvents());
    assertEquals(testInput.getTestName() + ": numDataEvents seen by producer", dstTestEvents.size(), clientStats.getTotalStats().getNumDataEvents());
    assertTrue(testInput.getTestName() + ": numSysEvents seen by producer", emitterStats.getTotalStats().getNumSysEvents() != 0);
    assertEquals(testInput.getTestName() + ": numSysEvents seen by writer", emitterStats.getTotalStats().getNumSysEvents(), streamStats.getTotalStats().getNumSysEvents());
    assertEquals(testInput.getTestName() + ": numSysEvents seen by reader", streamStats.getTotalStats().getNumSysEvents(), clientStats.getTotalStats().getNumSysEvents());
    // check integrity of each event received
    for (int i = 0; i < numEvents; ++i) {
        DbusEvent src = srcTestEvents.get(i);
        DbusEvent dst = dstTestEvents.get(i);
        assertEquals(testInput.getTestName() + ": src/dst srcIds of event #" + i, src.getSourceId(), dst.getSourceId());
        assertEquals(testInput.getTestName() + ": src/dst values of event #" + i, src.value(), dst.value());
    }
    return true;
}

Example 40 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class ReadEventsTestParams method testStatsMinMaxScn.

@Test
public void testStatsMinMaxScn() throws Exception {
    //Src Event producer
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    EventBufferTestInput input = new EventBufferTestInput();
    int numEvents = 10000;
    long startScn = 1000;
    int numWinScn = 10;
    //set sharedBufferSize to a value much smaller than total size required
    input.setNumEvents(numEvents).setWindowSize(numEvents / numWinScn).setSharedBufferSize(numEvents / 5).setStagingBufferSize(numEvents / 5).setIndexSize(numEvents / 10).setIndividualBufferSize(numEvents / 2).setBatchSize(numEvents / 5).setProducerBufferSize(numEvents / 2).setPayloadSize(100).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
    input.setTestName("testStatsMinMaxScn");
    DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
    DbusEventGenerator evGen = new DbusEventGenerator(startScn);
    if (evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), true, srcTestEvents) <= 0) {
        fail();
        return;
    }
    //sleep 10 ms;
    int eventSize = srcTestEvents.get(0).size();
    DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize, input.getIndividualBufferSize() * eventSize, input.getIndexSize() * eventSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
    DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats);
    Thread tEmitter = new Thread(eventProducer);
    tEmitter.start();
    tEmitter.join();
    //sleep 10 ms;
    int msDelay = 10;
    Thread.sleep(msDelay);
    long min = (numWinScn - 3) * input.getWindowSize() + startScn;
    long max = numWinScn * input.getWindowSize() + startScn;
    //note : event generator generates events such that a one second lag exists between the latest event and prev event
    long expectedRange = (max - min) + input.getWindowSize() - 1;
    System.out.printf("Total timespan = %d\n", (srcTestEvents.get(numEvents - 1).timestampInNanos() - srcTestEvents.get(0).timestampInNanos()) / NANOSECONDS);
    System.out.printf("prevScn=%d\n", emitterStats.getTotalStats().getPrevScn());
    System.out.printf("min = %d , max=%d  buf=%d ,%d\n", emitterStats.getTotalStats().getMinScn(), emitterStats.getTotalStats().getMaxScn(), prodEventBuffer.getMinScn(), prodEventBuffer.lastWrittenScn());
    System.out.printf("timespan=%d , timeSinceLastEvent = %d , timeSinceLastAccess %d\n", emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS, emitterStats.getTotalStats().getTimeSinceLastEvent(), emitterStats.getTotalStats().getTimeSinceLastAccess());
    assertEquals(numEvents, srcTestEvents.size());
    assertEquals(numEvents, emitterStats.getTotalStats().getNumDataEvents());
    assertEquals(min, emitterStats.getTotalStats().getMinScn());
    assertEquals(max, emitterStats.getTotalStats().getMaxScn());
    assertEquals(min - input.getWindowSize(), emitterStats.getTotalStats().getPrevScn());
    assertEquals(emitterStats.getTotalStats().getSizeDataEvents() * numEvents, numEvents * eventSize);
    long tsSpanInSec = emitterStats.getTotalStats().getTimeSpan() / MILLISECONDS;
    assertEquals(expectedRange, tsSpanInSec);
    long tsSinceLastEvent = emitterStats.getTotalStats().getTimeSinceLastEvent();
    assertTrue(tsSinceLastEvent >= msDelay);
    assertTrue(emitterStats.getTotalStats().getTimeLag() >= 0);
    assertTrue(emitterStats.getTotalStats().getMinTimeLag() >= 0);
    assertTrue(emitterStats.getTotalStats().getMaxTimeLag() >= 0);
    DbusEventBuffer readEventBuffer = new DbusEventBuffer(getConfig(numEvents * 2L * eventSize, numEvents * eventSize, (numEvents / 10) * eventSize, numEvents * 2 * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
    //check streaming
    Vector<Long> seenScns = new Vector<Long>();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    //case: where sinceScn < min , > prevScn ; return the entire buffer
    cp.setWindowScn(min - 1);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
    int expectedNumWin = (int) (max - min) / input.getWindowSize() + 1;
    assertEquals(expectedNumWin, seenScns.size());
    assertEquals(Long.valueOf(min), seenScns.get(0));
    assertEquals(Long.valueOf(max), seenScns.get(expectedNumWin - 1));
    //case : where sinceScn < prevScn ; exception thrown;
    cp.setWindowScn(startScn);
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(0, seenScns.size());
    //case: where sinceScn < min , = prevScn ; offset=-1 ,return the entire buffer
    cp.setWindowScn(prodEventBuffer.getPrevScn());
    cp.setWindowOffset(-1);
    seenScns.clear();
    readEventBuffer.clear();
    streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns);
    assertEquals(expectedNumWin, seenScns.size());
    assertEquals(Long.valueOf(min), seenScns.get(0));
    assertEquals(Long.valueOf(max), seenScns.get(expectedNumWin - 1));
    //case: where sinceScn < min , = prevScn ; offset=0 ,return nothing
    cp.setWindowScn(prodEventBuffer.getPrevScn());
    cp.setWindowOffset(0);
    seenScns.clear();
    readEventBuffer.clear();
    assertFalse(streamWriterReader(prodEventBuffer, numEvents * 2 * eventSize, cp, "scn", readEventBuffer, seenScns));
    assertEquals(0, seenScns.size());
}