Examples with DbusEventsTotalStats com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats used on opensource projects

Example 21 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestAggregatedDbusEventStatsCollector method runAggregateTestStats.

void runAggregateTestStats(int n) {
    try {
        DbusEventsStatisticsCollector aggregateEventStatsCollectors = new AggregatedDbusEventsStatisticsCollector(0, "eventsInbound", true, true, null);
        //collection of n+1 stats collectors;
        StatsCollectors<DbusEventsStatisticsCollector> eventStatsCollectors = new StatsCollectors<DbusEventsStatisticsCollector>(aggregateEventStatsCollectors);
        //add new individual stats collectors
        int maxEventsInWindow = 10;
        StatsWriter[] nStatsWriters = createStatsWriters(n, maxEventsInWindow);
        for (StatsWriter sw : nStatsWriters) {
            eventStatsCollectors.addStatsCollector(sw.getStatsName(), sw.getEventsStatsCollector());
        }
        //aggregator thread; 250 ms poll time
        GlobalStatsCalc agg = new GlobalStatsCalc(10);
        agg.registerStatsCollector(eventStatsCollectors);
        Thread aggThread = new Thread(agg);
        aggThread.start();
        //start writers
        for (StatsWriter sw : nStatsWriters) {
            sw.start();
        }
        //Let the writers start
        Thread.sleep(1000);
        long startTimeMs = System.currentTimeMillis();
        //5s
        long durationInMs = 5 * 1000;
        DbusEventsTotalStats globalStats = aggregateEventStatsCollectors.getTotalStats();
        long prevValue = 0, prevSize = 0;
        while (System.currentTimeMillis() < (startTimeMs + durationInMs)) {
            //constraint checks;
            //check that readers don't have partial updates or get initialized
            long value = globalStats.getNumDataEvents();
            long size = globalStats.getSizeDataEvents();
            Assert.assertTrue(value > 0);
            if (prevValue > 0 && (value != prevValue)) {
                Assert.assertTrue(size != prevSize);
                prevValue = value;
                prevSize = size;
            }
            Assert.assertTrue(globalStats.getMaxSeenWinScn() > 0);
            Thread.sleep(RngUtils.randomPositiveInt() % 10 + 1);
        }
        //shut down
        for (StatsWriter sw : nStatsWriters) {
            sw.shutdown();
            sw.interrupt();
        }
        //Give a chance to catch up
        Thread.sleep(1000);
        agg.halt();
        aggThread.interrupt();
        //final tally aggregatedEventTotalStats = sum of all individual statsWriter objects in a thread free way
        AggregatedDbusEventsTotalStats myTotalStats = new AggregatedDbusEventsTotalStats(StatsWriter.OWNERID, "mytotal", true, false, null);
        for (DbusEventsStatisticsCollector s : eventStatsCollectors.getStatsCollectors()) {
            DbusEventsTotalStats writerStat = s.getTotalStats();
            //obviously - we assume this is correct here. we want to check that the updates happen correctly in a concurrent setting
            myTotalStats.mergeStats(writerStat);
        }
        LOG.info("global = " + globalStats.getNumDataEvents() + " Sigma writers=" + myTotalStats.getNumDataEvents());
        Assert.assertEquals("NumDataEvents mismatch for n = " + n, globalStats.getNumDataEvents(), myTotalStats.getNumDataEvents());
        Assert.assertEquals("MaxSeenWinScn mismatch for n = " + n, globalStats.getMaxSeenWinScn(), myTotalStats.getMaxSeenWinScn());
    } catch (InterruptedException e) {
        Assert.assertTrue(false);
    }
}

Example 22 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ContainerStatsRequestProcessor method processEventsPeerStats.

private void processEventsPeerStats(DbusEventsStatisticsCollector statsCollector, String prefix, DatabusRequest request) throws IOException, RequestProcessingException {
    if (null == statsCollector)
        return;
    String category = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    String client = category.substring(prefix.length());
    DbusEventsTotalStats clientStats = statsCollector.getPeerStats(client);
    if (null == clientStats) {
        throw new InvalidRequestParamValueException(request.getName(), prefix, client);
    }
    writeJsonObjectToResponse(clientStats, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(clientStats, request);
    }
}

Example 23 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ContainerStatsRequestProcessor method processEventsSourceStats.

private void processEventsSourceStats(DbusEventsStatisticsCollector statsCollector, String prefix, DatabusRequest request) throws IOException, RequestProcessingException {
    if (null == statsCollector)
        return;
    String category = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    String sourceIdStr = category.substring(prefix.length());
    int sourceId = -1;
    try {
        sourceId = Integer.valueOf(sourceIdStr);
    } catch (NumberFormatException nfe) {
        throw new InvalidRequestParamValueException("bad srcId:" + request.getName(), prefix, sourceIdStr);
    }
    DbusEventsTotalStats sourceStats = null;
    sourceStats = statsCollector.getSourceStats(sourceId);
    if (null == sourceStats) {
        LOG.warn("no stats for this srcId: " + request.getName() + "prefix=" + prefix + "source ids " + sourceIdStr);
        sourceStats = new DbusEventsTotalStats(0, sourceIdStr, false, false, null);
    }
    writeJsonObjectToResponse(sourceStats, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(sourceStats, request);
    }
}

Example 24 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDbusEventBufferMult method testMultiPPartionStreamFromLatest.

@Test
public void testMultiPPartionStreamFromLatest() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 1, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 2, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
    CheckpointMult cpMult = new CheckpointMult();
    for (int i = 0; i < 3; ++i) {
        Checkpoint cp = new Checkpoint();
        cp.setFlexible();
        cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
        cpMult.addCheckpoint(p[i], cp);
    }
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    // Set streamFromLatestScn == true
    reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 3);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
    // Only the last window is returned in each of the partitions
    assertEquals(ppartStats[0].getNumDataEvents(), 1);
    assertEquals(ppartStats[1].getNumDataEvents(), 2);
    assertEquals(ppartStats[2].getNumDataEvents(), 3);
    assertEquals(ppartStats[0].getNumSysEvents(), 1);
    assertEquals(ppartStats[1].getNumSysEvents(), 1);
    assertEquals(ppartStats[2].getNumSysEvents(), 1);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1 + 2 + 3));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1 + 1 + 1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}

Example 25 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDbusEventBufferMult method testSinglePPartionStreamFromLatest.

@Test
public void testSinglePPartionStreamFromLatest() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]) };
    CheckpointMult cpMult = new CheckpointMult();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cpMult.addCheckpoint(p[0], cp);
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    // Set streamFromLatestScn == true
    reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 1);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats() };
    // Only the last window is returned in each of the partitions
    assertEquals(ppartStats[0].getNumDataEvents(), 1);
    assertEquals(ppartStats[0].getNumSysEvents(), 1);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), ppartStats[0].getTimeLag());
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), ppartStats[0].getTimeLag());
}