Examples with DbusEventsTotalStats com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats used on opensource projects

Example 11 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ClientStatsRequestProcessor method processEventsPeerStats.

private void processEventsPeerStats(DbusEventsStatisticsCollector statsCollector, String prefix, DatabusRequest request) throws IOException, RequestProcessingException {
    if (null == statsCollector)
        return;
    String category = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    String client = category.substring(prefix.length());
    DbusEventsTotalStats clientStats = statsCollector.getPeerStats(client);
    if (null == clientStats) {
        throw new InvalidRequestParamValueException(request.getName(), prefix, client);
    }
    writeJsonObjectToResponse(clientStats, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(clientStats, request);
    }
}

Example 12 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ClientStatsRequestProcessor method processEventsTotalStats.

private void processEventsTotalStats(DbusEventsStatisticsCollector statsCollector, DatabusRequest request) throws IOException {
    if (null == statsCollector)
        return;
    DbusEventsTotalStats totalStatsMBean = statsCollector.getTotalStats();
    if (null == totalStatsMBean)
        return;
    writeJsonObjectToResponse(totalStatsMBean, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(totalStatsMBean, request);
    }
}

Example 13 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class RelayContainerStatsRequestProcessor method processStats.

private void processStats(StatsCollectors<DbusEventsStatisticsCollector> globalStatsCollector, DbusEventStatsCollectorsPartitioner resourceGroupStatsCollector, String prefix, DatabusRequest request) throws IOException, RequestProcessingException {
    String reqPathStr = request.getParams().getProperty(DatabusRequest.PATH_PARAM_NAME);
    String reqPathSuffix = reqPathStr.substring(prefix.length());
    // allow DBNAME/partitionid for REST api
    reqPathSuffix = reqPathSuffix.replace('/', ':');
    DbusEventsTotalStats sourceStats = null;
    if (reqPathSuffix.contains(":")) {
        // This is a request for a specific partition
        if (null != globalStatsCollector) {
            DbusEventsStatisticsCollector s = globalStatsCollector.getStatsCollector(reqPathSuffix);
            sourceStats = (s == null) ? null : s.getTotalStats();
        }
    } else {
        // This is a request at DB aggregate level
        if (null != resourceGroupStatsCollector) {
            StatsCollectors<DbusEventsStatisticsCollector> c = resourceGroupStatsCollector.getDBStatsCollector(reqPathSuffix);
            if (null != c)
                sourceStats = c.getStatsCollector().getTotalStats();
        }
    }
    if (null == sourceStats) {
        LOG.warn("No Stats for this source=" + request.getName() + ", prefix=" + prefix + ", DB/Physical Partition String=" + reqPathSuffix);
        sourceStats = new DbusEventsTotalStats(0, reqPathSuffix, false, false, null);
    }
    writeJsonObjectToResponse(sourceStats, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(sourceStats, request);
    }
}

Example 14 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingSlowConsumer.

@Test
public /**
	 * Slow consumer; results in getting SCN not found exception ; as chained relay overwrites buffer .
	 * Makes sure chained relay overwrites buffer;just like main buffer
	 */
void testRelayChainingSlowConsumer() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1012, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay with only 1 MB buffer
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1013, chainedRelayPort, 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        // create slow consumer with 100ms delay
        CountingConsumer countingConsumer = new CountingConsumer(500);
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 1000, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // start slow client
        cr.start();
        // let it run for 20 seconds
        Thread.sleep(20 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 100 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        Assert.assertTrue(stats.getNumSysEvents() > 0);
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(stats.getMinScn() < statsChained.getMinScn());
        Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
        Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
        Assert.assertTrue(countingConsumer.getNumErrors() > 0);
        Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 2);
        Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 15 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingConnectFailure.

@Test
public /** Client and chained relay start before relay, simulating unavailable relay and retries at chained relay and client
	 *
	 */
void testRelayChainingConnectFailure() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1004, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1005, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // Let them try for 5seconds
        Thread.sleep(5 * 1000);
        r1.start();
        // Let the relay run for 10s
        Thread.sleep(10 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 5 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertEquals(stats.getNumDataEvents(), countingConsumer.getNumDataEvents());
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(stats.getNumSysEvents(), countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}