Examples with TopicList org.apache.rocketmq.common.protocol.body.TopicList used on opensource projects

Example 26 with TopicList

use of org.apache.rocketmq.common.protocol.body.TopicList in project rocketmq-externals by apache.

the class RmqSourceReplicator method fetchTargetTopics.

public Set<String> fetchTargetTopics() throws RemotingException, MQClientException, InterruptedException {
    String targetCluster = this.replicatorConfig.getTargetCluster();
    TopicList targetTopics = this.targetMQAdminExt.fetchTopicsByCLuster(targetCluster);
    return targetTopics.getTopicList();
}

Example 27 with TopicList

use of org.apache.rocketmq.common.protocol.body.TopicList in project rocketmq-externals by apache.

the class RmqSourceReplicator method buildRoute.

public void buildRoute() {
    List<Pattern> patterns = new ArrayList<Pattern>();
    String srcCluster = this.replicatorConfig.getSrcCluster();
    try {
        Set<String> targetTopicSet = fetchTargetTopics();
        for (String topic : this.replicatorConfig.getWhiteList()) {
            Pattern pattern = Pattern.compile(topic);
            patterns.add(pattern);
        }
        TopicList topics = srcMQAdminExt.fetchAllTopicList();
        for (String topic : topics.getTopicList()) {
            if (topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
                continue;
            }
            if (!syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
                continue;
            }
            if (!syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
                continue;
            }
            for (Pattern pattern : patterns) {
                Matcher matcher = pattern.matcher(topic);
                if (matcher.matches()) {
                    String targetTopic = generateTargetTopic(topic);
                    if (!targetTopicSet.contains(targetTopic)) {
                        ensureTargetTopic(topic, targetTopic);
                    }
                    // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
                    // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
                    // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
                    List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
                    Set<String> brokerNameSet = new HashSet<String>();
                    for (BrokerData b : brokerList) {
                        brokerNameSet.add(b.getBrokerName());
                    }
                    TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
                    if (!topicRouteMap.containsKey(topic)) {
                        topicRouteMap.put(topic, new HashSet<>(16));
                    }
                    for (QueueData qd : topicRouteData.getQueueDatas()) {
                        if (brokerNameSet.contains(qd.getBrokerName())) {
                            for (int i = 0; i < qd.getReadQueueNums(); i++) {
                                TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, targetTopic);
                                topicRouteMap.get(topic).add(taskTopicInfo);
                            }
                        }
                    }
                }
            }
        }
    } catch (Exception e) {
        log.error("Fetch topic list error.", e);
    }
}

Example 28 with TopicList

use of org.apache.rocketmq.common.protocol.body.TopicList in project rocketmq-externals by apache.

the class DashboardCollectTask method collectTopic.

@Scheduled(cron = "30 0/1 * * * ?")
@MultiMQAdminCmdMethod(timeoutMillis = 5000)
public void collectTopic() {
    if (!rmqConfigure.isEnableDashBoardCollect()) {
        return;
    }
    Date date = new Date();
    Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        TopicList topicList = mqAdminExt.fetchAllTopicList();
        Set<String> topicSet = topicList.getTopicList();
        for (String topic : topicSet) {
            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
                continue;
            }
            TopicRouteData topicRouteData = mqAdminExt.examineTopicRouteInfo(topic);
            GroupList groupList = mqAdminExt.queryTopicConsumeByWho(topic);
            double inTPS = 0;
            long inMsgCntToday = 0;
            double outTPS = 0;
            long outMsgCntToday = 0;
            for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
                if (masterAddr != null) {
                    try {
                        stopwatch.start();
                        log.info("start time: {}", stopwatch.toString());
                        BrokerStatsData bsd = mqAdminExt.viewBrokerStatsData(masterAddr, BrokerStatsManager.TOPIC_PUT_NUMS, topic);
                        stopwatch.stop();
                        log.info("stop time : {}", stopwatch.toString());
                        stopwatch.reset();
                        inTPS += bsd.getStatsMinute().getTps();
                        inMsgCntToday += StatsAllSubCommand.compute24HourSum(bsd);
                    } catch (Exception e) {
                    // throw Throwables.propagate(e);
                    }
                }
            }
            if (groupList != null && !groupList.getGroupList().isEmpty()) {
                for (String group : groupList.getGroupList()) {
                    for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                        String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
                        if (masterAddr != null) {
                            try {
                                String statsKey = String.format("%s@%s", topic, group);
                                BrokerStatsData bsd = mqAdminExt.viewBrokerStatsData(masterAddr, BrokerStatsManager.GROUP_GET_NUMS, statsKey);
                                outTPS += bsd.getStatsMinute().getTps();
                                outMsgCntToday += StatsAllSubCommand.compute24HourSum(bsd);
                            } catch (Exception e) {
                            // throw Throwables.propagate(e);
                            }
                        }
                    }
                }
            }
            List<String> list;
            try {
                list = dashboardCollectService.getTopicMap().get(topic);
            } catch (ExecutionException e) {
                throw Throwables.propagate(e);
            }
            if (null == list) {
                list = Lists.newArrayList();
            }
            list.add(date.getTime() + "," + new BigDecimal(inTPS).setScale(5, BigDecimal.ROUND_HALF_UP) + "," + inMsgCntToday + "," + new BigDecimal(outTPS).setScale(5, BigDecimal.ROUND_HALF_UP) + "," + outMsgCntToday);
            dashboardCollectService.getTopicMap().put(topic, list);
        }
        log.debug("Topic Collected Data in memory = {}" + JsonUtil.obj2String(dashboardCollectService.getTopicMap().asMap()));
    } catch (Exception err) {
        throw Throwables.propagate(err);
    }
}

Example 29 with TopicList

use of org.apache.rocketmq.common.protocol.body.TopicList in project rocketmq-externals by apache.

the class TopicServiceImplTest method fetchAllTopicList.

@Test
public void fetchAllTopicList() throws Exception {
    TopicList topicList = topicService.fetchAllTopicList();
    Assert.assertNotNull(topicList);
    Assert.assertTrue(CollectionUtils.isNotEmpty(topicList.getTopicList()));
    Assert.assertTrue(topicList.getTopicList().contains(TEST_CONSOLE_TOPIC));
}

Example 30 with TopicList

use of org.apache.rocketmq.common.protocol.body.TopicList in project rocketmq by apache.

the class MonitorService method doMonitorWork.

public void doMonitorWork() throws RemotingException, MQClientException, InterruptedException {
    long beginTime = System.currentTimeMillis();
    this.monitorListener.beginRound();
    TopicList topicList = defaultMQAdminExt.fetchAllTopicList();
    for (String topic : topicList.getTopicList()) {
        if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
            String consumerGroup = topic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());
            try {
                this.reportUndoneMsgs(consumerGroup);
            } catch (Exception e) {
            // log.error("reportUndoneMsgs Exception", e);
            }
            try {
                this.reportConsumerRunningInfo(consumerGroup);
            } catch (Exception e) {
            // log.error("reportConsumerRunningInfo Exception", e);
            }
        }
    }
    this.monitorListener.endRound();
    long spentTimeMills = System.currentTimeMillis() - beginTime;
    log.info("Execute one round monitor work, spent timemills: {}", spentTimeMills);
}