use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.
the class JmxShutdownThread method doShutdown.
protected void doShutdown() {
unregisterShutdownHook();
LOG.info("Initializing shutdown for serving container: " + _containerStaticConfig.getId());
if (null != _jmxShutdownThread && Thread.State.NEW == _jmxShutdownThread.getState()) {
try {
Runtime.getRuntime().removeShutdownHook(_jmxShutdownThread);
_jmxShutdownThread.start();
} catch (IllegalStateException ise) {
LOG.error("Error removing shutdown hook", ise);
}
}
if (null != _nettyShutdownThread && Thread.State.NEW == _nettyShutdownThread.getState()) {
try {
Runtime.getRuntime().removeShutdownHook(_nettyShutdownThread);
_nettyShutdownThread.start();
} catch (IllegalStateException ise) {
LOG.error("Error removing shutdown hook", ise);
}
}
if (_globalStatsMerger != null && !_globalStatsMerger.isHalted()) {
_globalStatsMerger.halt();
_globalStatsThread.interrupt();
}
// unregister all mbeans
getContainerStatsCollector().unregisterMBeans();
getInboundEventStatisticsCollector().unregisterMBeans();
getOutboundEventStatisticsCollector().unregisterMBeans();
for (DbusEventsStatisticsCollector coll : _inBoundStatsCollectors.getStatsCollectors()) {
coll.unregisterMBeans();
}
for (DbusEventsStatisticsCollector coll : _outBoundStatsCollectors.getStatsCollectors()) {
coll.unregisterMBeans();
}
_componentAdmin.unregisterAsMBeans();
LOG.info("joining shutdown threads");
long startTime = System.currentTimeMillis();
long timeRemaining = SHUTDOWN_TIMEOUT_MS;
while (null != _jmxShutdownThread && _jmxShutdownThread.isAlive() && timeRemaining > 0) {
try {
_jmxShutdownThread.join(timeRemaining);
} catch (InterruptedException ie) {
}
timeRemaining = SHUTDOWN_TIMEOUT_MS - (System.currentTimeMillis() - startTime);
}
LOG.info("JMX shutdown for container " + _containerStaticConfig.getId() + ":" + (null != _jmxShutdownThread ? !_jmxShutdownThread.isAlive() : true) + "; ms remaining: " + timeRemaining);
while (null != _nettyShutdownThread && _nettyShutdownThread.isAlive() && timeRemaining > 0) {
try {
_nettyShutdownThread.join(timeRemaining);
} catch (InterruptedException ie) {
}
timeRemaining = SHUTDOWN_TIMEOUT_MS - (System.currentTimeMillis() - startTime);
}
LOG.info("Netty shutdown for container " + _containerStaticConfig.getId() + ":" + (null != _nettyShutdownThread ? !_nettyShutdownThread.isAlive() : true) + "; ms remaining: " + timeRemaining);
LOG.info("Done with shutdown for serving container: " + _containerStaticConfig.getId());
}
use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.
the class DatabusHttpClientImpl method unregisterMbeans.
/**
*
*/
protected void unregisterMbeans() {
getHttpStatsCollector().unregisterMBeans();
getBootstrapEventsStatsCollector().unregisterMBeans();
for (DbusEventsStatisticsCollector b : _bootstrapEventsStats.getStatsCollectors()) {
b.unregisterMBeans();
}
for (DatabusSourcesConnection conn : _relayConnections) {
conn.unregisterMbeans();
}
}
use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.
the class TestDbusEventBufferMult method testMultiPPartionStreamFromLatest.
@Test
public void testMultiPPartionStreamFromLatest() throws Exception {
createBufMult();
PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
//generate a bunch of windows for 3 partitions
int windowsNum = 10;
for (int i = 1; i <= windowsNum; ++i) {
DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
buf.startEvents();
byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i, null);
buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
buf.startEvents();
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i + 1, null);
buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
buf.startEvents();
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i + 2, null);
}
String[] pnames = new String[p.length];
int count = 0;
for (PhysicalPartition ip : p) {
pnames[count++] = ip.toSimpleString();
}
StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
CheckpointMult cpMult = new CheckpointMult();
for (int i = 0; i < 3; ++i) {
Checkpoint cp = new Checkpoint();
cp.setFlexible();
cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
cpMult.addCheckpoint(p[i], cp);
}
DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
WritableByteChannel writeChannel = Channels.newChannel(baos);
// Set streamFromLatestScn == true
reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
writeChannel.close();
baos.close();
//make sure we got the physical partition names right
List<String> ppartNames = statsColl.getStatsCollectorKeys();
assertEquals(ppartNames.size(), 3);
HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
for (String ppartName : ppartNames) {
assertTrue(expectedPPartNames.contains(ppartName));
}
//verify event counts per partition
DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
// Only the last window is returned in each of the partitions
assertEquals(ppartStats[0].getNumDataEvents(), 1);
assertEquals(ppartStats[1].getNumDataEvents(), 2);
assertEquals(ppartStats[2].getNumDataEvents(), 3);
assertEquals(ppartStats[0].getNumSysEvents(), 1);
assertEquals(ppartStats[1].getNumSysEvents(), 1);
assertEquals(ppartStats[2].getNumSysEvents(), 1);
assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1 + 2 + 3));
assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1 + 1 + 1));
assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}
use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.
the class TestDbusEventBufferMult method testSinglePPartionStreamFromLatest.
@Test
public void testSinglePPartionStreamFromLatest() throws Exception {
createBufMult();
PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition() };
//generate a bunch of windows for 3 partitions
int windowsNum = 10;
for (int i = 1; i <= windowsNum; ++i) {
DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
buf.startEvents();
byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i, null);
}
String[] pnames = new String[p.length];
int count = 0;
for (PhysicalPartition ip : p) {
pnames[count++] = ip.toSimpleString();
}
StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]) };
CheckpointMult cpMult = new CheckpointMult();
Checkpoint cp = new Checkpoint();
cp.setFlexible();
cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
cpMult.addCheckpoint(p[0], cp);
DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
WritableByteChannel writeChannel = Channels.newChannel(baos);
// Set streamFromLatestScn == true
reader.streamEvents(true, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
writeChannel.close();
baos.close();
//make sure we got the physical partition names right
List<String> ppartNames = statsColl.getStatsCollectorKeys();
assertEquals(ppartNames.size(), 1);
HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString()));
for (String ppartName : ppartNames) {
assertTrue(expectedPPartNames.contains(ppartName));
}
//verify event counts per partition
DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats() };
// Only the last window is returned in each of the partitions
assertEquals(ppartStats[0].getNumDataEvents(), 1);
assertEquals(ppartStats[0].getNumSysEvents(), 1);
assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), (1));
assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), (1));
assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), ppartStats[0].getTimeLag());
assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), ppartStats[0].getTimeLag());
}
use of com.linkedin.databus.core.monitoring.mbean.DbusEventsStatisticsCollector in project databus by linkedin.
the class TestDbusEventBufferMult method testMultiPPartionStreamStats.
@Test
public void testMultiPPartionStreamStats() throws Exception {
createBufMult();
PhysicalPartition[] p = { _pConfigs[0].getPhysicalPartition(), _pConfigs[1].getPhysicalPartition(), _pConfigs[2].getPhysicalPartition() };
//generate a bunch of windows for 3 partitions
int windowsNum = 10;
for (int i = 1; i <= windowsNum; ++i) {
DbusEventBufferAppendable buf = _eventBufferMult.getDbusEventBufferAppendable(p[0]);
buf.startEvents();
byte[] schema = "abcdefghijklmnop".getBytes(Charset.defaultCharset());
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 100, (short) 0, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i, null);
buf = _eventBufferMult.getDbusEventBufferAppendable(p[1]);
buf.startEvents();
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i + 1, null);
buf = _eventBufferMult.getDbusEventBufferAppendable(p[2]);
buf.startEvents();
assertTrue(buf.appendEvent(new DbusEventKey(1), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[100], false, null));
assertTrue(buf.appendEvent(new DbusEventKey(2), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
assertTrue(buf.appendEvent(new DbusEventKey(3), (short) 101, (short) 2, System.currentTimeMillis() * 1000000, (short) 2, schema, new byte[10], false, null));
buf.endEvents(100 * i + 2, null);
}
String[] pnames = new String[p.length];
int count = 0;
for (PhysicalPartition ip : p) {
pnames[count++] = ip.toSimpleString();
}
StatsCollectors<DbusEventsStatisticsCollector> statsColl = createStats(pnames);
PhysicalPartitionKey[] pkeys = { new PhysicalPartitionKey(p[0]), new PhysicalPartitionKey(p[1]), new PhysicalPartitionKey(p[2]) };
CheckpointMult cpMult = new CheckpointMult();
for (int i = 0; i < 3; ++i) {
Checkpoint cp = new Checkpoint();
cp.setFlexible();
cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
cpMult.addCheckpoint(p[i], cp);
}
DbusEventBufferBatchReadable reader = _eventBufferMult.getDbusEventBufferBatchReadable(cpMult, Arrays.asList(pkeys), statsColl);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
WritableByteChannel writeChannel = Channels.newChannel(baos);
reader.streamEvents(false, 1000000, writeChannel, Encoding.BINARY, new AllowAllDbusFilter());
writeChannel.close();
baos.close();
//make sure we got the physical partition names right
List<String> ppartNames = statsColl.getStatsCollectorKeys();
assertEquals(ppartNames.size(), 3);
HashSet<String> expectedPPartNames = new HashSet<String>(Arrays.asList(p[0].toSimpleString(), p[1].toSimpleString(), p[2].toSimpleString()));
for (String ppartName : ppartNames) {
assertTrue(expectedPPartNames.contains(ppartName));
}
//verify event counts per partition
DbusEventsTotalStats[] ppartStats = { statsColl.getStatsCollector(p[0].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[1].toSimpleString()).getTotalStats(), statsColl.getStatsCollector(p[2].toSimpleString()).getTotalStats() };
assertEquals(ppartStats[0].getNumDataEvents(), windowsNum);
assertEquals(ppartStats[1].getNumDataEvents(), windowsNum * 2);
assertEquals(ppartStats[2].getNumDataEvents(), windowsNum * 3);
assertEquals(ppartStats[0].getNumSysEvents(), windowsNum);
assertEquals(ppartStats[1].getNumSysEvents(), windowsNum);
assertEquals(ppartStats[2].getNumSysEvents(), windowsNum);
assertEquals(statsColl.getStatsCollector().getTotalStats().getNumDataEvents(), windowsNum * (1 + 2 + 3));
assertEquals(statsColl.getStatsCollector().getTotalStats().getNumSysEvents(), windowsNum * 3);
assertEquals(statsColl.getStatsCollector().getTotalStats().getMaxTimeLag(), Math.max(ppartStats[0].getTimeLag(), Math.max(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
assertEquals(statsColl.getStatsCollector().getTotalStats().getMinTimeLag(), Math.min(ppartStats[0].getTimeLag(), Math.min(ppartStats[1].getTimeLag(), ppartStats[2].getTimeLag())));
}
Aggregations