Examples with ConsumeStats org.apache.rocketmq.common.admin.ConsumeStats used on opensource projects

Example 21 with ConsumeStats

use of org.apache.rocketmq.common.admin.ConsumeStats in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AdminBrokerProcessor method getConsumeStats.

private RemotingCommand getConsumeStats(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    final GetConsumeStatsRequestHeader requestHeader = (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
    ConsumeStats consumeStats = new ConsumeStats();
    Set<String> topics = new HashSet<String>();
    if (UtilAll.isBlank(requestHeader.getTopic())) {
        topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
    } else {
        topics.add(requestHeader.getTopic());
    }
    for (String topic : topics) {
        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
        if (null == topicConfig) {
            log.warn("consumeStats, topic config not exist, {}", topic);
            continue;
        }
        /**
         */
        {
            SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
            if (// 
            null == findSubscriptionData && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
                log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", requestHeader.getConsumerGroup(), topic);
                continue;
            }
        }
        for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
            MessageQueue mq = new MessageQueue();
            mq.setTopic(topic);
            mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
            mq.setQueueId(i);
            OffsetWrapper offsetWrapper = new OffsetWrapper();
            long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
            if (brokerOffset < 0)
                brokerOffset = 0;
            long consumerOffset = // 
            this.brokerController.getConsumerOffsetManager().queryOffset(// 
            requestHeader.getConsumerGroup(), // 
            topic, i);
            if (consumerOffset < 0)
                consumerOffset = 0;
            offsetWrapper.setBrokerOffset(brokerOffset);
            offsetWrapper.setConsumerOffset(consumerOffset);
            long timeOffset = consumerOffset - 1;
            if (timeOffset >= 0) {
                long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
                if (lastTimestamp > 0) {
                    offsetWrapper.setLastTimestamp(lastTimestamp);
                }
            }
            consumeStats.getOffsetTable().put(mq, offsetWrapper);
        }
        double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
        consumeTps += consumeStats.getConsumeTps();
        consumeStats.setConsumeTps(consumeTps);
    }
    byte[] body = consumeStats.encode();
    response.setBody(body);
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}

Example 22 with ConsumeStats

use of org.apache.rocketmq.common.admin.ConsumeStats in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQAdminExtTest method init.

@BeforeClass
public static void init() throws Exception {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);
    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);
    properties.setProperty("maxMessageSize", "5000000");
    properties.setProperty("flushDelayOffsetInterval", "15000");
    properties.setProperty("serverSocketRcvBufSize", "655350");
    when(mQClientAPIImpl.getBrokerConfig(anyString(), anyLong())).thenReturn(properties);
    Set<String> topicSet = new HashSet<>();
    topicSet.add("topic_one");
    topicSet.add("topic_two");
    topicList.setTopicList(topicSet);
    when(mQClientAPIImpl.getTopicListFromNameServer(anyLong())).thenReturn(topicList);
    List<BrokerData> brokerDatas = new ArrayList<>();
    HashMap<Long, String> brokerAddrs = new HashMap<>();
    brokerAddrs.put(1234l, "127.0.0.1:10911");
    BrokerData brokerData = new BrokerData();
    brokerData.setCluster("default-cluster");
    brokerData.setBrokerName("default-broker");
    brokerData.setBrokerAddrs(brokerAddrs);
    brokerDatas.add(brokerData);
    topicRouteData.setBrokerDatas(brokerDatas);
    topicRouteData.setQueueDatas(new ArrayList<QueueData>());
    topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    HashMap<String, String> result = new HashMap<>();
    result.put("id", "1234");
    result.put("brokerName", "default-broker");
    kvTable.setTable(result);
    when(mQClientAPIImpl.getBrokerRuntimeInfo(anyString(), anyLong())).thenReturn(kvTable);
    HashMap<String, BrokerData> brokerAddrTable = new HashMap<>();
    brokerAddrTable.put("default-broker", brokerData);
    brokerAddrTable.put("broker-test", new BrokerData());
    clusterInfo.setBrokerAddrTable(brokerAddrTable);
    clusterInfo.setClusterAddrTable(new HashMap<String, Set<String>>());
    when(mQClientAPIImpl.getBrokerClusterInfo(anyLong())).thenReturn(clusterInfo);
    when(mQClientAPIImpl.cleanExpiredConsumeQueue(anyString(), anyLong())).thenReturn(true);
    Set<String> clusterList = new HashSet<>();
    clusterList.add("default-cluster-one");
    clusterList.add("default-cluster-two");
    when(mQClientAPIImpl.getClusterList(anyString(), anyLong())).thenReturn(clusterList);
    GroupList groupList = new GroupList();
    HashSet<String> groups = new HashSet<>();
    groups.add("consumer-group-one");
    groups.add("consumer-group-two");
    groupList.setGroupList(groups);
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    when(mQClientAPIImpl.queryTopicConsumeByWho(anyString(), anyString(), anyLong())).thenReturn(groupList);
    SubscriptionGroupWrapper subscriptionGroupWrapper = new SubscriptionGroupWrapper();
    ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptions = new ConcurrentHashMap<>();
    SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
    subscriptionGroupConfig.setConsumeBroadcastEnable(true);
    subscriptionGroupConfig.setBrokerId(1234);
    subscriptionGroupConfig.setGroupName("Consumer-group-one");
    subscriptions.put("Consumer-group-one", subscriptionGroupConfig);
    subscriptionGroupWrapper.setSubscriptionGroupTable(subscriptions);
    when(mQClientAPIImpl.getAllSubscriptionGroup(anyString(), anyLong())).thenReturn(subscriptionGroupWrapper);
    String topicListConfig = "topicListConfig";
    when(mQClientAPIImpl.getKVConfigValue(anyString(), anyString(), anyLong())).thenReturn(topicListConfig);
    KVTable kvTable = new KVTable();
    HashMap<String, String> kv = new HashMap<>();
    kv.put("broker-name", "broker-one");
    kv.put("cluster-name", "default-cluster");
    kvTable.setTable(kv);
    when(mQClientAPIImpl.getKVListByNamespace(anyString(), anyLong())).thenReturn(kvTable);
    ConsumeStats consumeStats = new ConsumeStats();
    consumeStats.setConsumeTps(1234);
    MessageQueue messageQueue = new MessageQueue();
    OffsetWrapper offsetWrapper = new OffsetWrapper();
    HashMap<MessageQueue, OffsetWrapper> stats = new HashMap<>();
    stats.put(messageQueue, offsetWrapper);
    consumeStats.setOffsetTable(stats);
    when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), anyString(), anyLong())).thenReturn(consumeStats);
    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    connections.add(new Connection());
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
    ProducerConnection producerConnection = new ProducerConnection();
    Connection connection = new Connection();
    connection.setClientAddr("127.0.0.1:9898");
    connection.setClientId("PID_12345");
    HashSet<Connection> connectionSet = new HashSet<Connection>();
    connectionSet.add(connection);
    producerConnection.setConnectionSet(connectionSet);
    when(mQClientAPIImpl.getProducerConnectionList(anyString(), anyString(), anyLong())).thenReturn(producerConnection);
    when(mQClientAPIImpl.wipeWritePermOfBroker(anyString(), anyString(), anyLong())).thenReturn(6);
    TopicStatsTable topicStatsTable = new TopicStatsTable();
    topicStatsTable.setOffsetTable(new HashMap<MessageQueue, TopicOffset>());
    Map<String, Map<MessageQueue, Long>> consumerStatus = new HashMap<>();
    when(mQClientAPIImpl.invokeBrokerToGetConsumerStatus(anyString(), anyString(), anyString(), anyString(), anyLong())).thenReturn(consumerStatus);
    List<QueueTimeSpan> queueTimeSpanList = new ArrayList<>();
    when(mQClientAPIImpl.queryConsumeTimeSpan(anyString(), anyString(), anyString(), anyLong())).thenReturn(queueTimeSpanList);
    ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo();
    consumerRunningInfo.setJstack("test");
    consumerRunningInfo.setMqTable(new TreeMap<MessageQueue, ProcessQueueInfo>());
    consumerRunningInfo.setStatusTable(new TreeMap<String, ConsumeStatus>());
    consumerRunningInfo.setSubscriptionSet(new TreeSet<SubscriptionData>());
    when(mQClientAPIImpl.getConsumerRunningInfo(anyString(), anyString(), anyString(), anyBoolean(), anyLong())).thenReturn(consumerRunningInfo);
}

Example 23 with ConsumeStats

use of org.apache.rocketmq.common.admin.ConsumeStats in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MonitorServiceTest method init.

@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, RemotingException, MQClientException, InterruptedException, MQBrokerException {
    monitorConfig = new MonitorConfig();
    monitorListener = new DefaultMonitorListener();
    defaultMQPullConsumer = mock(DefaultMQPullConsumer.class);
    defaultMQPushConsumer = mock(DefaultMQPushConsumer.class);
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);
    monitorService = new MonitorService(monitorConfig, monitorListener, null);
    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);
    field = MonitorService.class.getDeclaredField("defaultMQAdminExt");
    field.setAccessible(true);
    field.set(monitorService, defaultMQAdminExt);
    field = MonitorService.class.getDeclaredField("defaultMQPullConsumer");
    field.setAccessible(true);
    field.set(monitorService, defaultMQPullConsumer);
    field = MonitorService.class.getDeclaredField("defaultMQPushConsumer");
    field.setAccessible(true);
    field.set(monitorService, defaultMQPushConsumer);
    TopicList topicList = new TopicList();
    Set<String> topicSet = new HashSet<>();
    topicSet.add("topic_one");
    topicSet.add("topic_two");
    topicList.setTopicList(topicSet);
    when(mQClientAPIImpl.getTopicListFromNameServer(anyLong())).thenReturn(topicList);
    TopicRouteData topicRouteData = new TopicRouteData();
    List<BrokerData> brokerDatas = new ArrayList<>();
    HashMap<Long, String> brokerAddrs = new HashMap<>();
    brokerAddrs.put(1234l, "127.0.0.1:10911");
    BrokerData brokerData = new BrokerData();
    brokerData.setCluster("default-cluster");
    brokerData.setBrokerName("default-broker");
    brokerData.setBrokerAddrs(brokerAddrs);
    brokerDatas.add(brokerData);
    topicRouteData.setBrokerDatas(brokerDatas);
    topicRouteData.setQueueDatas(new ArrayList<QueueData>());
    topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    ConsumeStats consumeStats = new ConsumeStats();
    consumeStats.setConsumeTps(1234);
    MessageQueue messageQueue = new MessageQueue();
    OffsetWrapper offsetWrapper = new OffsetWrapper();
    HashMap<MessageQueue, OffsetWrapper> stats = new HashMap<>();
    stats.put(messageQueue, offsetWrapper);
    consumeStats.setOffsetTable(stats);
    when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), anyString(), anyLong())).thenReturn(consumeStats);
    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    Connection connection = new Connection();
    connection.setClientId("client_id");
    connection.setClientAddr("127.0.0.1:109111");
    connection.setLanguage(LanguageCode.JAVA);
    connection.setVersion(MQVersion.Version.V4_0_0_SNAPSHOT.ordinal());
    connections.add(connection);
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
    ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo();
    consumerRunningInfo.setJstack("test");
    consumerRunningInfo.setMqTable(new TreeMap<MessageQueue, ProcessQueueInfo>());
    consumerRunningInfo.setStatusTable(new TreeMap<String, ConsumeStatus>());
    consumerRunningInfo.setSubscriptionSet(new TreeSet<SubscriptionData>());
    Properties properties = new Properties();
    properties.put(ConsumerRunningInfo.PROP_CONSUME_TYPE, CONSUME_ACTIVELY);
    properties.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, System.currentTimeMillis());
    consumerRunningInfo.setProperties(properties);
    when(mQClientAPIImpl.getConsumerRunningInfo(anyString(), anyString(), anyString(), anyBoolean(), anyLong())).thenReturn(consumerRunningInfo);
}

Example 24 with ConsumeStats

use of org.apache.rocketmq.common.admin.ConsumeStats in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class ConsumerProgressSubCommandTest method init.

@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, InterruptedException, RemotingException, MQClientException, MQBrokerException {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);
    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);
    TopicRouteData topicRouteData = new TopicRouteData();
    List<BrokerData> brokerDatas = new ArrayList<>();
    HashMap<Long, String> brokerAddrs = new HashMap<>();
    brokerAddrs.put(1234l, "127.0.0.1:10911");
    BrokerData brokerData = new BrokerData();
    brokerData.setCluster("default-cluster");
    brokerData.setBrokerName("default-broker");
    brokerData.setBrokerAddrs(brokerAddrs);
    brokerDatas.add(brokerData);
    topicRouteData.setBrokerDatas(brokerDatas);
    topicRouteData.setQueueDatas(new ArrayList<QueueData>());
    topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    ConsumeStats consumeStats = new ConsumeStats();
    consumeStats.setConsumeTps(1234);
    MessageQueue messageQueue = new MessageQueue();
    OffsetWrapper offsetWrapper = new OffsetWrapper();
    HashMap<MessageQueue, OffsetWrapper> stats = new HashMap<>();
    stats.put(messageQueue, offsetWrapper);
    consumeStats.setOffsetTable(stats);
    when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), anyString(), anyLong())).thenReturn(consumeStats);
}

Example 25 with ConsumeStats

use of org.apache.rocketmq.common.admin.ConsumeStats in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class StatsAllSubCommand method printTopicDetail.

public static void printTopicDetail(final DefaultMQAdminExt admin, final String topic, final boolean activeTopic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
    TopicRouteData topicRouteData = admin.examineTopicRouteInfo(topic);
    GroupList groupList = admin.queryTopicConsumeByWho(topic);
    double inTPS = 0;
    long inMsgCntToday = 0;
    for (BrokerData bd : topicRouteData.getBrokerDatas()) {
        String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
        if (masterAddr != null) {
            try {
                BrokerStatsData bsd = admin.viewBrokerStatsData(masterAddr, BrokerStatsManager.TOPIC_PUT_NUMS, topic);
                inTPS += bsd.getStatsMinute().getTps();
                inMsgCntToday += compute24HourSum(bsd);
            } catch (Exception e) {
            }
        }
    }
    if (groupList != null && !groupList.getGroupList().isEmpty()) {
        for (String group : groupList.getGroupList()) {
            double outTPS = 0;
            long outMsgCntToday = 0;
            for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
                if (masterAddr != null) {
                    try {
                        String statsKey = String.format("%s@%s", topic, group);
                        BrokerStatsData bsd = admin.viewBrokerStatsData(masterAddr, BrokerStatsManager.GROUP_GET_NUMS, statsKey);
                        outTPS += bsd.getStatsMinute().getTps();
                        outMsgCntToday += compute24HourSum(bsd);
                    } catch (Exception e) {
                    }
                }
            }
            long accumulate = 0;
            try {
                ConsumeStats consumeStats = admin.examineConsumeStats(group, topic);
                if (consumeStats != null) {
                    accumulate = consumeStats.computeTotalDiff();
                    if (accumulate < 0) {
                        accumulate = 0;
                    }
                }
            } catch (Exception e) {
            }
            if (!activeTopic || (inMsgCntToday > 0) || (outMsgCntToday > 0)) {
                // 第二个参数默认是32，有时候消费者长度过长显示不去，修改为52
                System.out.printf("%-32s  %-52s %12d %11.2f %11.2f %14d %14d%n", UtilAll.frontStringAtLeast(topic, 32), // 有时候消费者长度过长显示不去，修改为52
                UtilAll.frontStringAtLeast(group, 52), accumulate, inTPS, outTPS, inMsgCntToday, outMsgCntToday);
            }
        }
    } else {
        if (!activeTopic || (inMsgCntToday > 0)) {
            // 第二个参数默认是32，有时候消费者长度过长显示不去，修改为52
            System.out.printf("%-32s  %-52s %12d %11.2f %11s %14d %14s%n", UtilAll.frontStringAtLeast(topic, 52), "", 0, inTPS, "", inMsgCntToday, "NO_CONSUMER");
        }
    }
}