Examples with BrokerData org.apache.rocketmq.common.protocol.route.BrokerData used on opensource projects

Example 76 with BrokerData

use of org.apache.rocketmq.common.protocol.route.BrokerData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MQAdminImpl method queryMessage.

/**
 * 通过断点查看rocketmq-console项目查询MessageId我们可以发现，只有offsetMsgId才是真正的这种规则的key，而msgId其实仅仅知识生成的UNIQ_KEY
 * 这也是为什么用MessageDecoder.decodeMessageId端口号超限问题的原因
 */
protected QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end, boolean isUniqKey) throws MQClientException, InterruptedException {
    TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic);
    if (null == topicRouteData) {
        this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
        topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic);
    }
    if (topicRouteData != null) {
        List<String> brokerAddrs = new LinkedList<String>();
        for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
            String addr = brokerData.selectBrokerAddr();
            if (addr != null) {
                brokerAddrs.add(addr);
            }
        }
        if (!brokerAddrs.isEmpty()) {
            final CountDownLatch countDownLatch = new CountDownLatch(brokerAddrs.size());
            final List<QueryResult> queryResultList = new LinkedList<QueryResult>();
            final ReadWriteLock lock = new ReentrantReadWriteLock(false);
            for (String addr : brokerAddrs) {
                try {
                    // 构建查询请求头信息
                    QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader();
                    requestHeader.setTopic(topic);
                    requestHeader.setKey(key);
                    requestHeader.setMaxNum(maxNum);
                    requestHeader.setBeginTimestamp(begin);
                    requestHeader.setEndTimestamp(end);
                    this.mQClientFactory.getMQClientAPIImpl().queryMessage(addr, requestHeader, timeoutMillis * 3, new InvokeCallback() {

                        @Override
                        public void operationComplete(ResponseFuture responseFuture) {
                            try {
                                RemotingCommand response = responseFuture.getResponseCommand();
                                if (response != null) {
                                    switch(response.getCode()) {
                                        case ResponseCode.SUCCESS:
                                            {
                                                QueryMessageResponseHeader responseHeader = null;
                                                try {
                                                    responseHeader = (QueryMessageResponseHeader) response.decodeCommandCustomHeader(QueryMessageResponseHeader.class);
                                                } catch (RemotingCommandException e) {
                                                    log.error("decodeCommandCustomHeader exception", e);
                                                    return;
                                                }
                                                List<MessageExt> wrappers = MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true);
                                                QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers);
                                                try {
                                                    lock.writeLock().lock();
                                                    queryResultList.add(qr);
                                                } finally {
                                                    lock.writeLock().unlock();
                                                }
                                                break;
                                            }
                                        default:
                                            log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark());
                                            break;
                                    }
                                } else {
                                    log.warn("getResponseCommand return null");
                                }
                            } finally {
                                countDownLatch.countDown();
                            }
                        }
                    }, isUniqKey);
                } catch (Exception e) {
                    log.warn("queryMessage exception", e);
                }
            }
            boolean ok = countDownLatch.await(timeoutMillis * 4, TimeUnit.MILLISECONDS);
            if (!ok) {
                log.warn("queryMessage, maybe some broker failed");
            }
            long indexLastUpdateTimestamp = 0;
            List<MessageExt> messageList = new LinkedList<MessageExt>();
            for (QueryResult qr : queryResultList) {
                if (qr.getIndexLastUpdateTimestamp() > indexLastUpdateTimestamp) {
                    indexLastUpdateTimestamp = qr.getIndexLastUpdateTimestamp();
                }
                for (MessageExt msgExt : qr.getMessageList()) {
                    if (isUniqKey) {
                        if (msgExt.getMsgId().equals(key)) {
                            if (messageList.size() > 0) {
                                if (messageList.get(0).getStoreTimestamp() > msgExt.getStoreTimestamp()) {
                                    messageList.clear();
                                    messageList.add(msgExt);
                                }
                            } else {
                                messageList.add(msgExt);
                            }
                        } else {
                            log.warn("queryMessage by uniqKey, find message key not matched, maybe hash duplicate {}", msgExt.toString());
                        }
                    } else {
                        String keys = msgExt.getKeys();
                        if (keys != null) {
                            boolean matched = false;
                            String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR);
                            if (keyArray != null) {
                                for (String k : keyArray) {
                                    if (key.equals(k)) {
                                        matched = true;
                                        break;
                                    }
                                }
                            }
                            if (matched) {
                                messageList.add(msgExt);
                            } else {
                                log.warn("queryMessage, find message key not matched, maybe hash duplicate {}", msgExt.toString());
                            }
                        }
                    }
                }
            }
            if (!messageList.isEmpty()) {
                return new QueryResult(indexLastUpdateTimestamp, messageList);
            } else {
                throw new MQClientException(ResponseCode.NO_MESSAGE, "query message by key finished, but no message.");
            }
        }
    }
    throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "The topic[" + topic + "] not matched route info");
}

Example 77 with BrokerData

use of org.apache.rocketmq.common.protocol.route.BrokerData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQPushConsumerImpl method queryConsumeTimeSpan.

public List<QueueTimeSpan> queryConsumeTimeSpan(final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
    List<QueueTimeSpan> queueTimeSpan = new ArrayList<QueueTimeSpan>();
    TopicRouteData routeData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000);
    for (BrokerData brokerData : routeData.getBrokerDatas()) {
        String addr = brokerData.selectBrokerAddr();
        queueTimeSpan.addAll(this.mQClientFactory.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, groupName(), 3000));
    }
    return queueTimeSpan;
}

Example 78 with BrokerData

use of org.apache.rocketmq.common.protocol.route.BrokerData in project rocketmq by apache.

the class ClusterListSubCommand method printClusterMoreStats.

private void printClusterMoreStats(final DefaultMQAdminExt defaultMQAdminExt) throws RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, InterruptedException, MQBrokerException {
    ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo();
    System.out.printf("%-16s  %-32s %14s %14s %14s %14s%n", "#Cluster Name", "#Broker Name", "#InTotalYest", "#OutTotalYest", "#InTotalToday", "#OutTotalToday");
    Iterator<Map.Entry<String, Set<String>>> itCluster = clusterInfoSerializeWrapper.getClusterAddrTable().entrySet().iterator();
    while (itCluster.hasNext()) {
        Map.Entry<String, Set<String>> next = itCluster.next();
        String clusterName = next.getKey();
        TreeSet<String> brokerNameSet = new TreeSet<String>();
        brokerNameSet.addAll(next.getValue());
        for (String brokerName : brokerNameSet) {
            BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName);
            if (brokerData != null) {
                Iterator<Map.Entry<Long, String>> itAddr = brokerData.getBrokerAddrs().entrySet().iterator();
                while (itAddr.hasNext()) {
                    Map.Entry<Long, String> next1 = itAddr.next();
                    long inTotalYest = 0;
                    long outTotalYest = 0;
                    long inTotalToday = 0;
                    long outTotalToday = 0;
                    try {
                        KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(next1.getValue());
                        String msgPutTotalYesterdayMorning = kvTable.getTable().get("msgPutTotalYesterdayMorning");
                        String msgPutTotalTodayMorning = kvTable.getTable().get("msgPutTotalTodayMorning");
                        String msgPutTotalTodayNow = kvTable.getTable().get("msgPutTotalTodayNow");
                        String msgGetTotalYesterdayMorning = kvTable.getTable().get("msgGetTotalYesterdayMorning");
                        String msgGetTotalTodayMorning = kvTable.getTable().get("msgGetTotalTodayMorning");
                        String msgGetTotalTodayNow = kvTable.getTable().get("msgGetTotalTodayNow");
                        inTotalYest = Long.parseLong(msgPutTotalTodayMorning) - Long.parseLong(msgPutTotalYesterdayMorning);
                        outTotalYest = Long.parseLong(msgGetTotalTodayMorning) - Long.parseLong(msgGetTotalYesterdayMorning);
                        inTotalToday = Long.parseLong(msgPutTotalTodayNow) - Long.parseLong(msgPutTotalTodayMorning);
                        outTotalToday = Long.parseLong(msgGetTotalTodayNow) - Long.parseLong(msgGetTotalTodayMorning);
                    } catch (Exception e) {
                    }
                    System.out.printf("%-16s  %-32s %14d %14d %14d %14d%n", clusterName, brokerName, inTotalYest, outTotalYest, inTotalToday, outTotalToday);
                }
            }
        }
        if (itCluster.hasNext()) {
            System.out.printf("");
        }
    }
}

Example 79 with BrokerData

use of org.apache.rocketmq.common.protocol.route.BrokerData in project rocketmq by apache.

the class ClusterListSubCommand method printClusterBaseInfo.

private void printClusterBaseInfo(final DefaultMQAdminExt defaultMQAdminExt) throws RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, InterruptedException, MQBrokerException {
    ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo();
    System.out.printf("%-16s  %-22s  %-4s  %-22s %-16s %19s %19s %10s %5s %6s%n", "#Cluster Name", "#Broker Name", "#BID", "#Addr", "#Version", "#InTPS(LOAD)", "#OutTPS(LOAD)", "#PCWait(ms)", "#Hour", "#SPACE");
    Iterator<Map.Entry<String, Set<String>>> itCluster = clusterInfoSerializeWrapper.getClusterAddrTable().entrySet().iterator();
    while (itCluster.hasNext()) {
        Map.Entry<String, Set<String>> next = itCluster.next();
        String clusterName = next.getKey();
        TreeSet<String> brokerNameSet = new TreeSet<String>();
        brokerNameSet.addAll(next.getValue());
        for (String brokerName : brokerNameSet) {
            BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName);
            if (brokerData != null) {
                Iterator<Map.Entry<Long, String>> itAddr = brokerData.getBrokerAddrs().entrySet().iterator();
                while (itAddr.hasNext()) {
                    Map.Entry<Long, String> next1 = itAddr.next();
                    double in = 0;
                    double out = 0;
                    String version = "";
                    String sendThreadPoolQueueSize = "";
                    String pullThreadPoolQueueSize = "";
                    String sendThreadPoolQueueHeadWaitTimeMills = "";
                    String pullThreadPoolQueueHeadWaitTimeMills = "";
                    String pageCacheLockTimeMills = "";
                    String earliestMessageTimeStamp = "";
                    String commitLogDiskRatio = "";
                    try {
                        KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(next1.getValue());
                        String putTps = kvTable.getTable().get("putTps");
                        String getTransferedTps = kvTable.getTable().get("getTransferedTps");
                        sendThreadPoolQueueSize = kvTable.getTable().get("sendThreadPoolQueueSize");
                        pullThreadPoolQueueSize = kvTable.getTable().get("pullThreadPoolQueueSize");
                        sendThreadPoolQueueSize = kvTable.getTable().get("sendThreadPoolQueueSize");
                        pullThreadPoolQueueSize = kvTable.getTable().get("pullThreadPoolQueueSize");
                        sendThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().get("sendThreadPoolQueueHeadWaitTimeMills");
                        pullThreadPoolQueueHeadWaitTimeMills = kvTable.getTable().get("pullThreadPoolQueueHeadWaitTimeMills");
                        pageCacheLockTimeMills = kvTable.getTable().get("pageCacheLockTimeMills");
                        earliestMessageTimeStamp = kvTable.getTable().get("earliestMessageTimeStamp");
                        commitLogDiskRatio = kvTable.getTable().get("commitLogDiskRatio");
                        version = kvTable.getTable().get("brokerVersionDesc");
                        {
                            String[] tpss = putTps.split(" ");
                            if (tpss.length > 0) {
                                in = Double.parseDouble(tpss[0]);
                            }
                        }
                        {
                            String[] tpss = getTransferedTps.split(" ");
                            if (tpss.length > 0) {
                                out = Double.parseDouble(tpss[0]);
                            }
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    double hour = 0.0;
                    double space = 0.0;
                    if (earliestMessageTimeStamp != null && earliestMessageTimeStamp.length() > 0) {
                        long mills = System.currentTimeMillis() - Long.valueOf(earliestMessageTimeStamp);
                        hour = mills / 1000.0 / 60.0 / 60.0;
                    }
                    if (commitLogDiskRatio != null && commitLogDiskRatio.length() > 0) {
                        space = Double.valueOf(commitLogDiskRatio);
                    }
                    System.out.printf("%-16s  %-22s  %-4s  %-22s %-16s %19s %19s %10s %5s %6s%n", clusterName, brokerName, next1.getKey(), next1.getValue(), version, String.format("%9.2f(%s,%sms)", in, sendThreadPoolQueueSize, sendThreadPoolQueueHeadWaitTimeMills), String.format("%9.2f(%s,%sms)", out, pullThreadPoolQueueSize, pullThreadPoolQueueHeadWaitTimeMills), pageCacheLockTimeMills, String.format("%2.2f", hour), String.format("%.4f", space));
                }
            }
        }
        if (itCluster.hasNext()) {
            System.out.printf("");
        }
    }
}

Example 80 with BrokerData

use of org.apache.rocketmq.common.protocol.route.BrokerData in project rocketmq by apache.

the class StatsAllSubCommand method printTopicDetail.

public static void printTopicDetail(final DefaultMQAdminExt admin, final String topic, final boolean activeTopic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
    TopicRouteData topicRouteData = admin.examineTopicRouteInfo(topic);
    GroupList groupList = admin.queryTopicConsumeByWho(topic);
    double inTPS = 0;
    long inMsgCntToday = 0;
    for (BrokerData bd : topicRouteData.getBrokerDatas()) {
        String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
        if (masterAddr != null) {
            try {
                BrokerStatsData bsd = admin.viewBrokerStatsData(masterAddr, BrokerStatsManager.TOPIC_PUT_NUMS, topic);
                inTPS += bsd.getStatsMinute().getTps();
                inMsgCntToday += compute24HourSum(bsd);
            } catch (Exception e) {
            }
        }
    }
    if (groupList != null && !groupList.getGroupList().isEmpty()) {
        for (String group : groupList.getGroupList()) {
            double outTPS = 0;
            long outMsgCntToday = 0;
            for (BrokerData bd : topicRouteData.getBrokerDatas()) {
                String masterAddr = bd.getBrokerAddrs().get(MixAll.MASTER_ID);
                if (masterAddr != null) {
                    try {
                        String statsKey = String.format("%s@%s", topic, group);
                        BrokerStatsData bsd = admin.viewBrokerStatsData(masterAddr, BrokerStatsManager.GROUP_GET_NUMS, statsKey);
                        outTPS += bsd.getStatsMinute().getTps();
                        outMsgCntToday += compute24HourSum(bsd);
                    } catch (Exception e) {
                    }
                }
            }
            long accumulate = 0;
            try {
                ConsumeStats consumeStats = admin.examineConsumeStats(group, topic);
                if (consumeStats != null) {
                    accumulate = consumeStats.computeTotalDiff();
                    if (accumulate < 0) {
                        accumulate = 0;
                    }
                }
            } catch (Exception e) {
            }
            if (!activeTopic || (inMsgCntToday > 0) || (outMsgCntToday > 0)) {
                System.out.printf("%-32s  %-32s %12d %11.2f %11.2f %14d %14d%n", UtilAll.frontStringAtLeast(topic, 32), UtilAll.frontStringAtLeast(group, 32), accumulate, inTPS, outTPS, inMsgCntToday, outMsgCntToday);
            }
        }
    } else {
        if (!activeTopic || (inMsgCntToday > 0)) {
            System.out.printf("%-32s  %-32s %12d %11.2f %11s %14d %14s%n", UtilAll.frontStringAtLeast(topic, 32), "", 0, inTPS, "", inMsgCntToday, "NO_CONSUMER");
        }
    }
}