Search in sources :

Example 31 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class JmxShutdownThread method doShutdown.

protected void doShutdown() {
    unregisterShutdownHook();
    LOG.info("Initializing shutdown for serving container: " + _containerStaticConfig.getId());
    if (null != _jmxShutdownThread && Thread.State.NEW == _jmxShutdownThread.getState()) {
        try {
            Runtime.getRuntime().removeShutdownHook(_jmxShutdownThread);
            _jmxShutdownThread.start();
        } catch (IllegalStateException ise) {
            LOG.error("Error removing shutdown hook", ise);
        }
    }
    if (null != _nettyShutdownThread && Thread.State.NEW == _nettyShutdownThread.getState()) {
        try {
            Runtime.getRuntime().removeShutdownHook(_nettyShutdownThread);
            _nettyShutdownThread.start();
        } catch (IllegalStateException ise) {
            LOG.error("Error removing shutdown hook", ise);
        }
    }
    if (_globalStatsMerger != null && !_globalStatsMerger.isHalted()) {
        _globalStatsMerger.halt();
        _globalStatsThread.interrupt();
    }
    // unregister all mbeans
    getContainerStatsCollector().unregisterMBeans();
    getInboundEventStatisticsCollector().unregisterMBeans();
    getOutboundEventStatisticsCollector().unregisterMBeans();
    for (DbusEventsStatisticsCollector coll : _inBoundStatsCollectors.getStatsCollectors()) {
        coll.unregisterMBeans();
    }
    for (DbusEventsStatisticsCollector coll : _outBoundStatsCollectors.getStatsCollectors()) {
        coll.unregisterMBeans();
    }
    _componentAdmin.unregisterAsMBeans();
    LOG.info("joining shutdown threads");
    long startTime = System.currentTimeMillis();
    long timeRemaining = SHUTDOWN_TIMEOUT_MS;
    while (null != _jmxShutdownThread && _jmxShutdownThread.isAlive() && timeRemaining > 0) {
        try {
            _jmxShutdownThread.join(timeRemaining);
        } catch (InterruptedException ie) {
        }
        timeRemaining = SHUTDOWN_TIMEOUT_MS - (System.currentTimeMillis() - startTime);
    }
    LOG.info("JMX shutdown for container " + _containerStaticConfig.getId() + ":" + (null != _jmxShutdownThread ? !_jmxShutdownThread.isAlive() : true) + "; ms remaining: " + timeRemaining);
    while (null != _nettyShutdownThread && _nettyShutdownThread.isAlive() && timeRemaining > 0) {
        try {
            _nettyShutdownThread.join(timeRemaining);
        } catch (InterruptedException ie) {
        }
        timeRemaining = SHUTDOWN_TIMEOUT_MS - (System.currentTimeMillis() - startTime);
    }
    LOG.info("Netty shutdown for container " + _containerStaticConfig.getId() + ":" + (null != _nettyShutdownThread ? !_nettyShutdownThread.isAlive() : true) + "; ms remaining: " + timeRemaining);
    LOG.info("Done with shutdown for serving container: " + _containerStaticConfig.getId());
}
Also used : DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector)

Example 32 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class DatabusHttpClientImpl method unregisterMbeans.

/**
   *
   */
protected void unregisterMbeans() {
    getHttpStatsCollector().unregisterMBeans();
    getBootstrapEventsStatsCollector().unregisterMBeans();
    for (DbusEventsStatisticsCollector b : _bootstrapEventsStats.getStatsCollectors()) {
        b.unregisterMBeans();
    }
    for (DatabusSourcesConnection conn : _relayConnections) {
        conn.unregisterMbeans();
    }
}
Also used : DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector)

Example 33 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class TestDbusEventBufferMult method testMultiPPartionStreamFromLatest.

@Test
public void testMultiPPartionStreamFromLatest() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 1, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 2, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
    CheckpointMult cpMult = new CheckpointMult();
    for (int i = 0; i < 3; ++i) {
        Checkpoint cp = new Checkpoint();
        cp.setFlexible();
        cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
        cpMult.addCheckpoint(p[i], cp);
    }
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    // Set streamFromLatestScn == true
    reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 3);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
    // Only the last window is returned in each of the partitions
    assertEquals(ppartStats[0].getNumDataEvents(), 1);
    assertEquals(ppartStats[1].getNumDataEvents(), 2);
    assertEquals(ppartStats[2].getNumDataEvents(), 3);
    assertEquals(ppartStats[0].getNumSysEvents(), 1);
    assertEquals(ppartStats[1].getNumSysEvents(), 1);
    assertEquals(ppartStats[2].getNumSysEvents(), 1);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1 + 2 + 3));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1 + 1 + 1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}
Also used : WritableByteChannel(java.nio.channels.WritableByteChannel) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector) ByteArrayOutputStream(java.io.ByteArrayOutputStream) AllowAllDbusFilter(com.linkedin.databus2.core.filter.AllowAllDbusFilter) PhysicalPartitionKey(com.linkedin.databus.core.DbusEventBufferMult.PhysicalPartitionKey) DbusEventsTotalStats(com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats) PhysicalPartition(com.linkedin.databus.core.data_model.PhysicalPartition) HashSet(java.util.HashSet) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 34 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class TestDbusEventBufferMult method testSinglePPartionStreamFromLatest.

@Test
public void testSinglePPartionStreamFromLatest() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]) };
    CheckpointMult cpMult = new CheckpointMult();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
    cpMult.addCheckpoint(p[0], cp);
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    // Set streamFromLatestScn == true
    reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 1);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats() };
    // Only the last window is returned in each of the partitions
    assertEquals(ppartStats[0].getNumDataEvents(), 1);
    assertEquals(ppartStats[0].getNumSysEvents(), 1);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), ppartStats[0].getTimeLag());
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), ppartStats[0].getTimeLag());
}
Also used : WritableByteChannel(java.nio.channels.WritableByteChannel) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector) ByteArrayOutputStream(java.io.ByteArrayOutputStream) AllowAllDbusFilter(com.linkedin.databus2.core.filter.AllowAllDbusFilter) PhysicalPartitionKey(com.linkedin.databus.core.DbusEventBufferMult.PhysicalPartitionKey) DbusEventsTotalStats(com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats) PhysicalPartition(com.linkedin.databus.core.data_model.PhysicalPartition) HashSet(java.util.HashSet) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 35 with DbusEventsStatisticsCollector

use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.

the class TestDbusEventBufferMult method testMultiPPartionStreamStats.

@Test
public void testMultiPPartionStreamStats() throws Exception {
    createBufMult();
    PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
    //generate a bunch of windows for 3 partitions
    int windowsNum = 10;
    for (int i = 1; i <= windowsNum; ++i) {
        DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
        buf.startEvents();
        byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 1, null);
        buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
        buf.startEvents();
        assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
        buf.endEvents(100 * i + 2, null);
    }
    String[] pnames = new String[p.length];
    int count = 0;
    for (PhysicalPartition ip : p) {
        pnames[count++] = ip.toSimpleString();
    }
    StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
    PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
    CheckpointMult cpMult = new CheckpointMult();
    for (int i = 0; i < 3; ++i) {
        Checkpoint cp = new Checkpoint();
        cp.setFlexible();
        cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
        cpMult.addCheckpoint(p[i], cp);
    }
    DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    WritableByteChannel writeChannel = Channels.newChannel(baos);
    reader.streamEvents(false, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
    writeChannel.close();
    baos.close();
    //make sure we got the physical partition names right
    List<String> ppartNames = statsColl.getStatsCollectorKeys();
    assertEquals(ppartNames.size(), 3);
    HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
    for (String ppartName : ppartNames) {
        assertTrue(expectedPPartNames.contains(ppartName));
    }
    //verify event counts per partition
    DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
    assertEquals(ppartStats[0].getNumDataEvents(), windowsNum);
    assertEquals(ppartStats[1].getNumDataEvents(), windowsNum * 2);
    assertEquals(ppartStats[2].getNumDataEvents(), windowsNum * 3);
    assertEquals(ppartStats[0].getNumSysEvents(), windowsNum);
    assertEquals(ppartStats[1].getNumSysEvents(), windowsNum);
    assertEquals(ppartStats[2].getNumSysEvents(), windowsNum);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), windowsNum * (1 + 2 + 3));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), windowsNum * 3);
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
    assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}
Also used : WritableByteChannel(java.nio.channels.WritableByteChannel) DbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector) AggregatedDbusEventsStatisticsCollector(com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector) ByteArrayOutputStream(java.io.ByteArrayOutputStream) AllowAllDbusFilter(com.linkedin.databus2.core.filter.AllowAllDbusFilter) PhysicalPartitionKey(com.linkedin.databus.core.DbusEventBufferMult.PhysicalPartitionKey) DbusEventsTotalStats(com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats) PhysicalPartition(com.linkedin.databus.core.data_model.PhysicalPartition) HashSet(java.util.HashSet) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Aggregations

DbusEventsStatisticsCollector (com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector)41 Test (org.testng.annotations.Test)22 Checkpoint (com.linkedin.databus.core.Checkpoint)13 DbusEventBuffer (com.linkedin.databus.core.DbusEventBuffer)11 RegisterResponseEntry (com.linkedin.databus2.core.container.request.RegisterResponseEntry)11 AggregatedDbusEventsStatisticsCollector (com.linkedin.databus.core.monitoring.mbean.AggregatedDbusEventsStatisticsCollector)10 Logger (org.apache.log4j.Logger)10 DbusEventsTotalStats (com.linkedin.databus.core.monitoring.mbean.DbusEventsTotalStats)9 ArrayList (java.util.ArrayList)9 HashMap (java.util.HashMap)9 DatabusSubscription (com.linkedin.databus.core.data_model.DatabusSubscription)8 PhysicalPartition (com.linkedin.databus.core.data_model.PhysicalPartition)8 ConditionCheck (com.linkedin.databus2.test.ConditionCheck)8 Vector (java.util.Vector)8 List (java.util.List)7 NettyHttpDatabusRelayConnection (com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnection)6 NettyHttpDatabusRelayConnectionInspector (com.linkedin.databus.client.netty.NettyHttpDatabusRelayConnectionInspector)6 UnifiedClientStats (com.linkedin.databus.client.pub.mbean.UnifiedClientStats)6 DbusEventInfo (com.linkedin.databus.core.DbusEventInfo)6 ConsumerCallbackStats (com.linkedin.databus.client.pub.mbean.ConsumerCallbackStats)5