Examples with DbusEventsTotalStats com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats used on opensource projects

Example 1 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ContainerStatsRequestProcessor method processEventsTotalStats.

private void processEventsTotalStats(DbusEventsStatisticsCollector statsCollector, DatabusRequest request) throws IOException {
    if (null == statsCollector)
        return;
    DbusEventsTotalStats totalStatsMBean = statsCollector.getTotalStats();
    if (null == totalStatsMBean)
        return;
    writeJsonObjectToResponse(totalStatsMBean, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(totalStatsMBean, request);
    }
}

Example 2 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayEventGenerator.

@Test
public void testRelayEventGenerator() throws InterruptedException, InvalidConfigException {
    DatabusRelayTestUtil.RelayRunner r1 = null;
    // Logger.getRootLogger().setLevel(Level.INFO);
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayEventGenerator");
    // log.setLevel(Level.DEBUG);
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        log.info("create main relay with random generator");
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 20;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = Utils.getAvailablePort(11993);
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay1, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("async starts");
        r1.start();
        log.info("start client in parallel");
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + relayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("terminating conditions");
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        long totalRunTime = 5000;
        long startTime = System.currentTimeMillis();
        do {
            log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
            Thread.sleep(1000);
        } while ((System.currentTimeMillis() - startTime) < totalRunTime);
        r1.pause();
        log.info("Sending pause to relay!");
        log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                boolean success = true;
                for (EventProducer p : relay1.getProducers()) {
                    if (!(success = success && p.isPaused()))
                        break;
                }
                return success;
            }
        }, "waiting for producers to pause", 4000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
                return countingConsumer.getNumWindows() == stats.getNumSysEvents();
            }
        }, "wait until client got all events or for maxTimeout", 64 * 1024, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + stats.getNumSysEvents());
        cr.shutdown(2000, log);
        log.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        log.info("Relay r1 stopped");
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 3 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelalyChainingPartialSubscribe.

@Test
public /**
 * Client consumes subset of sources from chained relay
 */
void testRelalyChainingPartialSubscribe() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1006, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1007, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = "com.linkedin.events.example.Settings";
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, -1, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // start relay r1
        r1.start();
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // Let the relay run for 10s
        Thread.sleep(10 * 1000);
        r1.pause();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 5 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 1);
        Assert.assertTrue(stats.getNumSysEvents() == countingConsumer.getNumWindows());
        Assert.assertTrue(stats.getNumDataEvents() == 2 * countingConsumer.getNumDataEvents());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 4 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingDelayedConsumer.

@Test
public /**
 * Chained relay overwrites buffer; consumer starts late; gets only the last n events that fit.
 * Makes sure chained relay overwrites buffer;just like main buffer
 */
void testRelayChainingDelayedConsumer() {
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1010, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay with only 1 MB buffer
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1011, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        resetSCN(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // let it run for 10 seconds
        Thread.sleep(10 * 1000);
        r1.pause();
        // start client after the relays have finished
        cr.start();
        // wait until client got all events or for maxTimeout;
        long maxTimeOutMs = 15 * 1000;
        long startTime = System.currentTimeMillis();
        DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        while (countingConsumer.getNumWindows() < stats.getNumSysEvents()) {
            Thread.sleep(500);
            // stats.getSizeDataEvents());
            if ((System.currentTimeMillis() - startTime) > maxTimeOutMs) {
                break;
            }
        }
        DbusEventsTotalStats statsChained = relay2.getInboundEventStatisticsCollector().getTotalStats();
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        LOG.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        Assert.assertTrue(stats.getNumDataEvents() == statsChained.getNumDataEvents());
        Assert.assertTrue(stats.getNumSysEvents() == statsChained.getNumSysEvents());
        Assert.assertTrue(stats.getNumDataEvents() > countingConsumer.getNumDataEvents());
        Assert.assertTrue(countingConsumer.getNumSources() == 2);
        Assert.assertTrue(stats.getNumSysEvents() > countingConsumer.getNumWindows());
    } catch (Exception e) {
        LOG.error("Exception: " + e);
        Assert.assertTrue(false);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2 }, cr);
    }
}

Example 5 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingSCNRegress.

@Test
public /**
 * Test regression of SCN  of chained relay
 */
void testRelayChainingSCNRegress() throws InvalidConfigException, InterruptedException {
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingSCNRegress");
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
    ClientRunner cr = null;
    // log.setLevel(Level.DEBUG);
    log.info("start");
    // DbusEventBuffer.LOG.setLevel(Level.DEBUG);
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 10;
        int largestEventSize = 512 * 1024;
        long largestWindowSize = 1 * 1024 * 1024;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1019, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        final DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1020, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotNull(relay1);
        Assert.assertNotNull(relay3);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        final DbusEventsTotalStats stats3 = relay3.getInboundEventStatisticsCollector().getTotalStats();
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, 500, eventRatePerSec, 0, largestEventSize, largestWindowSize, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        final DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1021, chainedRelayPort, 10 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotNull(relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, largestEventSize, 30 * 1000, 100, 15 * 1000, 1, true, largestEventSize / 10);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        Thread.sleep(10 * 1000);
        // start chained relay
        r2.start();
        // start client
        cr.start();
        // Pause r1;
        r1.pause();
        Thread.sleep(1000);
        long firstGenDataEvents = stats.getNumDataEvents();
        long firstGenMinScn = stats.getMinScn();
        long firstGenWindows = stats.getNumSysEvents();
        Assert.assertTrue(stats.getNumSysEvents() > 0);
        log.warn("relay1:  numDataEvents=" + firstGenDataEvents + " numWindows=" + firstGenWindows + " minScn=" + firstGenMinScn + " maxScn=" + stats.getMaxScn());
        Thread.sleep(4 * 1000);
        // clear the relay
        boolean s = r1.shutdown(2000);
        Assert.assertTrue(s);
        DbusEventsTotalStats stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
        long firstGenChainWindows = stats2.getNumSysEvents();
        log.warn("relay2:  numDataEvents=" + stats2.getNumDataEvents() + " numWindows=" + firstGenChainWindows + " minScn=" + stats2.getMinScn() + " maxScn=" + stats2.getMaxScn());
        Thread.sleep(2 * 1000);
        // restart relay
        r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
        r3.start();
        Thread.sleep(15 * 1000);
        r3.pause();
        Thread.sleep(35 * 1000);
        log.warn("relay3:  numDataEvents=" + stats3.getNumDataEvents() + " numWindows=" + stats3.getNumSysEvents() + " minScn=" + stats3.getMinScn() + " maxScn=" + stats3.getMaxScn());
        stats2 = relay2.getInboundEventStatisticsCollector().getTotalStats();
        log.warn("relay2b: numDataEvents=" + stats2.getNumDataEvents() + " numWindows=" + stats2.getNumSysEvents() + " minScn=" + stats2.getMinScn() + " maxScn=" + stats2.getMaxScn());
        log.warn("consumer: " + countingConsumer);
        // compare chained relays with 2 gens of tier 0 relays
        Assert.assertEquals(stats2.getMinScn(), firstGenMinScn);
        Assert.assertEquals(stats2.getMaxScn(), stats3.getMaxScn());
        // the total event windows seen by the chained relay will be state of consumption at first failure of relay1 minus 1 overlap window
        Assert.assertEquals(stats2.getNumSysEvents(), (firstGenChainWindows - 1) + stats3.getNumSysEvents());
        Assert.assertTrue(stats2.getNumDataEvents() > stats3.getNumDataEvents());
        // compare source to final client
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(stats2.getNumSysEvents(), countingConsumer.getNumWindows());
        boolean sorted = true;
        long prev = -1;
        log.info(" scn seq on consumer=");
        for (Long l : countingConsumer.getEndScns()) {
            sorted = sorted && (l >= prev);
            prev = l;
            log.info(l + " ");
            if (!sorted)
                break;
        }
        Assert.assertTrue(sorted);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
        log.info("end");
    }
}