Examples with TopicRouteData org.apache.rocketmq.common.protocol.route.TopicRouteData used on opensource projects

Example 46 with TopicRouteData

use of org.apache.rocketmq.common.protocol.route.TopicRouteData in project rocketmq-externals by apache.

the class DashboardCollectTask method collectTopic.

@Scheduled(cron = "30 0/1 * * * ?")
@MultiMQAdminCmdMethod(timeoutMillis = 5000)
public void collectTopic() {
    if (!rmqConfigure.isEnableDashBoardCollect()) {
        return;
    }
    Date date = new Date();
    Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        TopicList topicList = mqAdminExt.fetchAllTopicList();
        Set<String> topicSet = topicList.getTopicList();
        for (String topic : topicSet) {
            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
                continue;
            }
            TopicRouteData topicRouteData = mqAdminExt.examineTopicRouteInfo(topic);
            GroupList groupList = mqAdminExt.queryTopicConsumeByWho(topic);
            double inTPS = 0;
            long inMsgCntToday = 0;
            double outTPS = 0;
            long outMsgCntToday = 0;
            for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
                if (masterAddr != null) {
                    try {
                        stopwatch.start();
                        log.info("start time: {}", stopwatch.toString());
                        BrokerStatsData bsd = mqAdminExt.viewBrokerStatsData(masterAddr, BrokerStatsManager.TOPIC_PUT_NUMS, topic);
                        stopwatch.stop();
                        log.info("stop time : {}", stopwatch.toString());
                        stopwatch.reset();
                        inTPS += bsd.getStatsMinute().getTps();
                        inMsgCntToday += StatsAllSubCommand.compute24HourSum(bsd);
                    } catch (Exception e) {
                    // throw Throwables.propagate(e);
                    }
                }
            }
            if (groupList != null && !groupList.getGroupList().isEmpty()) {
                for (String group : groupList.getGroupList()) {
                    for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                        String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
                        if (masterAddr != null) {
                            try {
                                String statsKey = String.format("%s@%s", topic, group);
                                BrokerStatsData bsd = mqAdminExt.viewBrokerStatsData(masterAddr, BrokerStatsManager.GROUP_GET_NUMS, statsKey);
                                outTPS += bsd.getStatsMinute().getTps();
                                outMsgCntToday += StatsAllSubCommand.compute24HourSum(bsd);
                            } catch (Exception e) {
                            // throw Throwables.propagate(e);
                            }
                        }
                    }
                }
            }
            List<String> list;
            try {
                list = dashboardCollectService.getTopicMap().get(topic);
            } catch (ExecutionException e) {
                throw Throwables.propagate(e);
            }
            if (null == list) {
                list = Lists.newArrayList();
            }
            list.add(date.getTime() + "," + new BigDecimal(inTPS).setScale(5, BigDecimal.ROUND_HALF_UP) + "," + inMsgCntToday + "," + new BigDecimal(outTPS).setScale(5, BigDecimal.ROUND_HALF_UP) + "," + outMsgCntToday);
            dashboardCollectService.getTopicMap().put(topic, list);
        }
        log.debug("Topic Collected Data in memory = {}" + JsonUtil.obj2String(dashboardCollectService.getTopicMap().asMap()));
    } catch (Exception err) {
        throw Throwables.propagate(err);
    }
}

Example 47 with TopicRouteData

use of org.apache.rocketmq.common.protocol.route.TopicRouteData in project rocketmq-externals by apache.

the class HudiSinkConnector method buildRoute.

public void buildRoute() {
    String srcCluster = this.sinkConnectConfig.getSrcCluster();
    try {
        for (String topic : this.sinkConnectConfig.getWhiteList()) {
            // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
            // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
            // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
            List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
            Set<String> brokerNameSet = new HashSet<String>();
            for (BrokerData b : brokerList) {
                brokerNameSet.add(b.getBrokerName());
            }
            TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
            if (!topicRouteMap.containsKey(topic)) {
                topicRouteMap.put(topic, new HashSet<>(16));
            }
            for (QueueData qd : topicRouteData.getQueueDatas()) {
                if (brokerNameSet.contains(qd.getBrokerName())) {
                    for (int i = 0; i < qd.getReadQueueNums(); i++) {
                        MessageQueue taskTopicInfo = new MessageQueue(topic, qd.getBrokerName(), i);
                        topicRouteMap.get(topic).add(taskTopicInfo);
                    }
                }
            }
        }
    } catch (Exception e) {
        log.error("Fetch topic list error.", e);
    } finally {
        srcMQAdminExt.shutdown();
    }
}

Example 48 with TopicRouteData

use of org.apache.rocketmq.common.protocol.route.TopicRouteData in project rocketmq-externals by apache.

the class Utils method examineBrokerData.

public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic, String cluster) throws RemotingException, MQClientException, InterruptedException {
    List<BrokerData> brokerList = new ArrayList<>();
    TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
    if (topicRouteData.getBrokerDatas() != null) {
        for (BrokerData broker : topicRouteData.getBrokerDatas()) {
            if (StringUtils.equals(broker.getCluster(), cluster)) {
                brokerList.add(broker);
            }
        }
    }
    return brokerList;
}

Example 49 with TopicRouteData

use of org.apache.rocketmq.common.protocol.route.TopicRouteData in project rocketmq-externals by apache.

the class Utils method examineBrokerData.

public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic, String cluster) throws RemotingException, MQClientException, InterruptedException {
    List<BrokerData> brokerList = new ArrayList<>();
    TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
    if (topicRouteData.getBrokerDatas() != null) {
        for (BrokerData broker : topicRouteData.getBrokerDatas()) {
            if (StringUtils.equals(broker.getCluster(), cluster)) {
                brokerList.add(broker);
            }
        }
    }
    return brokerList;
}

Example 50 with TopicRouteData

use of org.apache.rocketmq.common.protocol.route.TopicRouteData in project rocketmq-externals by apache.

the class JdbcSinkConnector method buildRoute.

public void buildRoute() {
    String srcCluster = ((SinkDbConnectorConfig) this.dbConnectorConfig).getSrcCluster();
    try {
        for (String topic : ((SinkDbConnectorConfig) this.dbConnectorConfig).getWhiteList()) {
            // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
            // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
            // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
            List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
            Set<String> brokerNameSet = new HashSet<String>();
            for (BrokerData b : brokerList) {
                brokerNameSet.add(b.getBrokerName());
            }
            TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
            if (!topicRouteMap.containsKey(topic)) {
                topicRouteMap.put(topic, new HashSet<>(16));
            }
            for (QueueData qd : topicRouteData.getQueueDatas()) {
                if (brokerNameSet.contains(qd.getBrokerName())) {
                    for (int i = 0; i < qd.getReadQueueNums(); i++) {
                        TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, null);
                        topicRouteMap.get(topic).add(taskTopicInfo);
                    }
                }
            }
        }
    } catch (Exception e) {
        log.error("Fetch topic list error.", e);
    } finally {
        srcMQAdminExt.shutdown();
    }
}