Examples with DbusEventsTotalStats com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats used on opensource projects

Example 6 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayChainingRestartSCNOffset.

@Test
public /**
	 * Test relay chaining that utilizes restart SCN offset
	 */
void testRelayChainingRestartSCNOffset() throws InterruptedException, InvalidConfigException {
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayChainingRestartSCNOffset");
    log.info("start");
    DatabusRelayTestUtil.RelayRunner r1 = null, r2 = null, r3 = null;
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.Account", "com.linkedin.events.example.Settings" } };
        // create main relay with random generator
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 2;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = 11993;
        DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1016, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        // create chained relay
        PhysicalSourceConfig[] chainedSrcConfigs = new PhysicalSourceConfig[srcNames.length];
        int j = 0;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (j + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "localhost:" + relayPort, eventRatePerSec, 50, srcs);
            chainedSrcConfigs[j++] = src1;
        }
        int chainedRelayPort = relayPort + 1;
        DatabusRelayMain relay2 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1017, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertTrue(null != relay2);
        r2 = new DatabusRelayTestUtil.RelayRunner(relay2);
        resetSCN(relay2);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + chainedRelayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        // async starts for all components;
        r1.start();
        // start chained relay
        r2.start();
        // start client
        Thread.sleep(500);
        cr.start();
        r1.pause();
        // let it run for 10 seconds
        Thread.sleep(10 * 1000);
        // wait until client got all events or for maxTimeout;
        final long maxTimeOutMs = 5 * 1000;
        final DbusEventsTotalStats dbRelayStats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumWindows() == dbRelayStats.getNumSysEvents();
            }
        }, "consumer caught up", maxTimeOutMs, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + dbRelayStats.getNumSysEvents());
        log.info("numDataEvents=" + dbRelayStats.getNumDataEvents() + " numWindows=" + dbRelayStats.getNumSysEvents() + " size=" + dbRelayStats.getSizeDataEvents());
        Assert.assertEquals(dbRelayStats.getNumDataEvents(), countingConsumer.getNumDataEvents());
        Assert.assertEquals(countingConsumer.getNumSources(), 2);
        Assert.assertEquals(dbRelayStats.getNumSysEvents(), countingConsumer.getNumWindows());
        cr.shutdown();
        boolean s2 = r2.shutdown(2000);
        Assert.assertTrue(s2);
        Assert.assertTrue(!r2.isAlive());
        //start r3; new chained relay with restart SCN offset; we get some data;
        chainedSrcConfigs[0].setRestartScnOffset(dbRelayStats.getMaxScn() - dbRelayStats.getPrevScn());
        DatabusRelayMain relay3 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1018, chainedRelayPort, 1 * 1024 * 1024, chainedSrcConfigs, SCHEMA_REGISTRY_DIR);
        r3 = new DatabusRelayTestUtil.RelayRunner(relay3);
        r3.start();
        final DbusEventsTotalStats newChainedRlyStats = relay3.getInboundEventStatisticsCollector().getTotalStats();
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return newChainedRlyStats.getNumDataEvents() > 0;
            }
        }, "new chained relay running", 5000, log);
        log.info("Stats3= numDataEvents=" + newChainedRlyStats.getNumDataEvents() + " numWindows=" + newChainedRlyStats.getNumSysEvents() + " size=" + newChainedRlyStats.getSizeDataEvents());
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1, r2, r3 }, cr);
        log.info("end");
    }
}

Example 7 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testClientNoEventsResetConnection.

@Test
public /**
	 * test resetting connection when there is no events for some period of time
	 * @throws InterruptedException
	 * @throws InvalidConfigException
	 */
void testClientNoEventsResetConnection() throws InterruptedException, InvalidConfigException {
    LOG.setLevel(Level.ALL);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String srcName = "com.linkedin.events.example.Settings";
        // create main relay with random generator
        int eventRatePerSec = 10;
        PhysicalSourceConfig srcConfig = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) 1, DatabusRelayTestUtil.getPhysicalSrcName(srcName), "mock", 500, eventRatePerSec, new String[] { srcName });
        int relayPort = 11995;
        DatabusRelayMain relay = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, new PhysicalSourceConfig[] { srcConfig }, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay);
        // async starts
        r1.start();
        DbusEventsTotalStats stats = relay.getInboundEventStatisticsCollector().getTotalStats();
        // start client in parallel
        String srcSubscriptionString = srcName;
        String serverName = "localhost:" + relayPort;
        ResetsCountingConsumer countingConsumer = new ResetsCountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        // generate some events
        TestUtil.sleep(1000);
        // pause event generator
        // and wait untill all the events are consumed
        // but since it is less then timeout the connection should NOT reset
        LOG.info("Sending pause to relay!");
        r1.pause();
        TestUtil.sleep(4000);
        LOG.info("no events, time less then threshold. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
        Assert.assertEquals(countingConsumer.getNumResets(), 0);
        // generate more events, more time elapsed then the threshold, but since there are
        // events - NO reset
        r1.unpause();
        Thread.sleep(8000);
        LOG.info("some events, more time then timeout. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
        Assert.assertEquals(countingConsumer.getNumResets(), 0);
        // stop events
        r1.pause();
        //set threshold to 0 completely disabling the feature
        clientConn.getRelayPullThread().setNoEventsConnectionResetTimeSec(0);
        Thread.sleep(8000);
        LOG.info("no events, more time then timeout, but feature disabled. Events=" + countingConsumer.getNumDataEvents() + "; resets = " + countingConsumer.getNumResets());
        Assert.assertEquals(countingConsumer.getNumResets(), 0);
        // enable the feature, and sleep for timeout
        clientConn.getRelayPullThread().setNoEventsConnectionResetTimeSec(5);
        // now wait with no events
        LOG.info("pause the producer. sleep for 6 sec, should reset");
        TestUtil.sleep(6000);
        LOG.info("Client stats=" + countingConsumer);
        LOG.info("Num resets=" + countingConsumer.getNumResets());
        LOG.info("Event windows generated=" + stats.getNumSysEvents());
        Assert.assertEquals(countingConsumer.getNumResets(), 0, "0 resets");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        LOG.info("Relay r1 stopped");
        cr.shutdown();
        LOG.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 8 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class TestDatabusRelayMain method testRelayEventGenerator.

@Test
public void testRelayEventGenerator() throws InterruptedException, InvalidConfigException {
    DatabusRelayTestUtil.RelayRunner r1 = null;
    //Logger.getRootLogger().setLevel(Level.INFO);
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayEventGenerator");
    //log.setLevel(Level.DEBUG);
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        log.info("create main relay with random generator");
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 20;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = Utils.getAvailablePort(11993);
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay1, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("async starts");
        r1.start();
        log.info("start client in parallel");
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + relayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("terminating conditions");
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        long totalRunTime = 5000;
        long startTime = System.currentTimeMillis();
        do {
            log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
            Thread.sleep(1000);
        } while ((System.currentTimeMillis() - startTime) < totalRunTime);
        r1.pause();
        log.info("Sending pause to relay!");
        log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                boolean success = true;
                for (EventProducer p : relay1.getProducers()) {
                    if (!(success = success && p.isPaused()))
                        break;
                }
                return success;
            }
        }, "waiting for producers to pause", 4000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
                return countingConsumer.getNumWindows() == stats.getNumSysEvents();
            }
        }, "wait until client got all events or for maxTimeout", 64 * 1024, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + stats.getNumSysEvents());
        cr.shutdown(2000, log);
        log.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        log.info("Relay r1 stopped");
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 9 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ContainerStatsRequestProcessor method processEventsTotalStats.

private void processEventsTotalStats(DbusEventsStatisticsCollector statsCollector, DatabusRequest request) throws IOException {
    if (null == statsCollector)
        return;
    DbusEventsTotalStats totalStatsMBean = statsCollector.getTotalStats();
    if (null == totalStatsMBean)
        return;
    writeJsonObjectToResponse(totalStatsMBean, request);
    if (request.getRequestType() == HttpMethod.PUT || request.getRequestType() == HttpMethod.POST) {
        enableOrResetStatsMBean(totalStatsMBean, request);
    }
}

Example 10 with DbusEventsTotalStats

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats in project databus by linkedin.

the class ReadEventsTestParams method testScnIndexInvalidation.

@Test
public void testScnIndexInvalidation() throws Exception {
    Vector<DbusEvent> srcTestEvents = new Vector<DbusEvent>();
    EventBufferTestInput input = new EventBufferTestInput();
    int numEvents = 1000;
    int numScns = 10;
    int windowSize = numEvents / numScns;
    DbusEventGenerator evGen = new DbusEventGenerator();
    for (int i = 1; i < 2; ++i) {
        LOG.info("I=" + i);
        int bufferSize = (windowSize * 2);
        input.setNumEvents(numEvents).setWindowSize(windowSize).setSharedBufferSize(bufferSize).setStagingBufferSize(bufferSize).setIndexSize(bufferSize).setIndividualBufferSize(bufferSize).setBatchSize(bufferSize).setProducerBufferSize(bufferSize).setPayloadSize(10).setDeleteInterval(1).setProdQueuePolicy(QueuePolicy.OVERWRITE_ON_WRITE);
        DbusEventsStatisticsCollector emitterStats = new DbusEventsStatisticsCollector(1, "appenderStats", true, true, null);
        if (srcTestEvents.isEmpty()) {
            assertTrue(evGen.generateEvents(numEvents, input.getWindowSize(), 512, input.getPayloadSize(), false, srcTestEvents) > 0);
        }
        int eventSize = 0;
        for (DbusEvent e : srcTestEvents) {
            if (!e.isControlMessage()) {
                eventSize = e.size();
                break;
            }
        }
        LOG.info("event size=" + eventSize);
        int indexSize = (input.getIndexSize() * eventSize / 100);
        LOG.info("indexSize=" + indexSize);
        DbusEventBuffer prodEventBuffer = new DbusEventBuffer(getConfig(input.getProducerBufferSize() * eventSize + i, input.getIndividualBufferSize() * eventSize, indexSize, input.getStagingBufferSize() * eventSize, AllocationPolicy.HEAP_MEMORY, input.getProdQueuePolicy(), input.getProdBufferAssertLevel()));
        LOG.info("Allocated buffer size=" + prodEventBuffer.getAllocatedSize());
        DbusEventAppender eventProducer = new DbusEventAppender(srcTestEvents, prodEventBuffer, emitterStats, 0.600);
        Thread t = new Thread(eventProducer);
        t.start();
        t.join();
        DbusEventsTotalStats stats = emitterStats.getTotalStats();
        LOG.info("minScn=" + prodEventBuffer.getMinScn() + "totalEvents=" + stats.getNumDataEvents());
    //weak assertion ! just check it's not -1 for now
    //TODO! get this assertion to succeed
    //assertTrue(prodEventBuffer.getMinScn()!=-1);
    }
}