Examples with SubscriptionData org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData used on opensource projects

Example 41 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MonitorServiceTest method init.

@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, RemotingException, MQClientException, InterruptedException, MQBrokerException {
    monitorConfig = new MonitorConfig();
    monitorListener = new DefaultMonitorListener();
    defaultMQPullConsumer = mock(DefaultMQPullConsumer.class);
    defaultMQPushConsumer = mock(DefaultMQPushConsumer.class);
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);
    monitorService = new MonitorService(monitorConfig, monitorListener, null);
    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);
    field = MonitorService.class.getDeclaredField("defaultMQAdminExt");
    field.setAccessible(true);
    field.set(monitorService, defaultMQAdminExt);
    field = MonitorService.class.getDeclaredField("defaultMQPullConsumer");
    field.setAccessible(true);
    field.set(monitorService, defaultMQPullConsumer);
    field = MonitorService.class.getDeclaredField("defaultMQPushConsumer");
    field.setAccessible(true);
    field.set(monitorService, defaultMQPushConsumer);
    TopicList topicList = new TopicList();
    Set<String> topicSet = new HashSet<>();
    topicSet.add("topic_one");
    topicSet.add("topic_two");
    topicList.setTopicList(topicSet);
    when(mQClientAPIImpl.getTopicListFromNameServer(anyLong())).thenReturn(topicList);
    TopicRouteData topicRouteData = new TopicRouteData();
    List<BrokerData> brokerDatas = new ArrayList<>();
    HashMap<Long, String> brokerAddrs = new HashMap<>();
    brokerAddrs.put(1234l, "127.0.0.1:10911");
    BrokerData brokerData = new BrokerData();
    brokerData.setCluster("default-cluster");
    brokerData.setBrokerName("default-broker");
    brokerData.setBrokerAddrs(brokerAddrs);
    brokerDatas.add(brokerData);
    topicRouteData.setBrokerDatas(brokerDatas);
    topicRouteData.setQueueDatas(new ArrayList<QueueData>());
    topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    ConsumeStats consumeStats = new ConsumeStats();
    consumeStats.setConsumeTps(1234);
    MessageQueue messageQueue = new MessageQueue();
    OffsetWrapper offsetWrapper = new OffsetWrapper();
    HashMap<MessageQueue, OffsetWrapper> stats = new HashMap<>();
    stats.put(messageQueue, offsetWrapper);
    consumeStats.setOffsetTable(stats);
    when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), anyString(), anyLong())).thenReturn(consumeStats);
    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    Connection connection = new Connection();
    connection.setClientId("client_id");
    connection.setClientAddr("127.0.0.1:109111");
    connection.setLanguage(LanguageCode.JAVA);
    connection.setVersion(MQVersion.Version.V4_0_0_SNAPSHOT.ordinal());
    connections.add(connection);
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
    ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo();
    consumerRunningInfo.setJstack("test");
    consumerRunningInfo.setMqTable(new TreeMap<MessageQueue, ProcessQueueInfo>());
    consumerRunningInfo.setStatusTable(new TreeMap<String, ConsumeStatus>());
    consumerRunningInfo.setSubscriptionSet(new TreeSet<SubscriptionData>());
    Properties properties = new Properties();
    properties.put(ConsumerRunningInfo.PROP_CONSUME_TYPE, CONSUME_ACTIVELY);
    properties.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, System.currentTimeMillis());
    consumerRunningInfo.setProperties(properties);
    when(mQClientAPIImpl.getConsumerRunningInfo(anyString(), anyString(), anyString(), anyBoolean(), anyLong())).thenReturn(consumerRunningInfo);
}

Example 42 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class ConsumerFilterManager method register.

public void register(final String consumerGroup, final Collection<SubscriptionData> subList) {
    for (SubscriptionData subscriptionData : subList) {
        register(subscriptionData.getTopic(), consumerGroup, subscriptionData.getSubString(), subscriptionData.getExpressionType(), subscriptionData.getSubVersion());
    }
    // make illegal topic dead.
    Collection<ConsumerFilterData> groupFilterData = getByGroup(consumerGroup);
    Iterator<ConsumerFilterData> iterator = groupFilterData.iterator();
    while (iterator.hasNext()) {
        ConsumerFilterData filterData = iterator.next();
        boolean exist = false;
        for (SubscriptionData subscriptionData : subList) {
            if (subscriptionData.getTopic().equals(filterData.getTopic())) {
                exist = true;
                break;
            }
        }
        if (!exist && !filterData.isDead()) {
            filterData.setDeadTime(System.currentTimeMillis());
            log.info("Consumer filter changed: {}, make illegal topic dead:{}", consumerGroup, filterData);
        }
    }
}

Example 43 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AdminBrokerProcessor method getConsumeStats.

private RemotingCommand getConsumeStats(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    final GetConsumeStatsRequestHeader requestHeader = (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
    ConsumeStats consumeStats = new ConsumeStats();
    Set<String> topics = new HashSet<String>();
    if (UtilAll.isBlank(requestHeader.getTopic())) {
        topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
    } else {
        topics.add(requestHeader.getTopic());
    }
    for (String topic : topics) {
        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
        if (null == topicConfig) {
            log.warn("consumeStats, topic config not exist, {}", topic);
            continue;
        }
        /**
         */
        {
            SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
            if (// 
            null == findSubscriptionData && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
                log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", requestHeader.getConsumerGroup(), topic);
                continue;
            }
        }
        for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
            MessageQueue mq = new MessageQueue();
            mq.setTopic(topic);
            mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
            mq.setQueueId(i);
            OffsetWrapper offsetWrapper = new OffsetWrapper();
            long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
            if (brokerOffset < 0)
                brokerOffset = 0;
            long consumerOffset = // 
            this.brokerController.getConsumerOffsetManager().queryOffset(// 
            requestHeader.getConsumerGroup(), // 
            topic, i);
            if (consumerOffset < 0)
                consumerOffset = 0;
            offsetWrapper.setBrokerOffset(brokerOffset);
            offsetWrapper.setConsumerOffset(consumerOffset);
            long timeOffset = consumerOffset - 1;
            if (timeOffset >= 0) {
                long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
                if (lastTimestamp > 0) {
                    offsetWrapper.setLastTimestamp(lastTimestamp);
                }
            }
            consumeStats.getOffsetTable().put(mq, offsetWrapper);
        }
        double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
        consumeTps += consumeStats.getConsumeTps();
        consumeStats.setConsumeTps(consumeTps);
    }
    byte[] body = consumeStats.encode();
    response.setBody(body);
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}

Example 44 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AdminBrokerProcessor method queryConsumeQueue.

private RemotingCommand queryConsumeQueue(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    QueryConsumeQueueRequestHeader requestHeader = (QueryConsumeQueueRequestHeader) request.decodeCommandCustomHeader(QueryConsumeQueueRequestHeader.class);
    RemotingCommand response = RemotingCommand.createResponseCommand(null);
    ConsumeQueue consumeQueue = this.brokerController.getMessageStore().getConsumeQueue(requestHeader.getTopic(), requestHeader.getQueueId());
    if (consumeQueue == null) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark(String.format("%d@%s is not exist!", requestHeader.getQueueId(), requestHeader.getTopic()));
        return response;
    }
    QueryConsumeQueueResponseBody body = new QueryConsumeQueueResponseBody();
    response.setCode(ResponseCode.SUCCESS);
    response.setBody(body.encode());
    body.setMaxQueueIndex(consumeQueue.getMaxOffsetInQueue());
    body.setMinQueueIndex(consumeQueue.getMinOffsetInQueue());
    MessageFilter messageFilter = null;
    if (requestHeader.getConsumerGroup() != null) {
        SubscriptionData subscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), requestHeader.getTopic());
        body.setSubscriptionData(subscriptionData);
        if (subscriptionData == null) {
            body.setFilterData(String.format("%s@%s is not online!", requestHeader.getConsumerGroup(), requestHeader.getTopic()));
        } else {
            ConsumerFilterData filterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(), requestHeader.getConsumerGroup());
            body.setFilterData(JSON.toJSONString(filterData, true));
            messageFilter = new ExpressionMessageFilter(subscriptionData, filterData, this.brokerController.getConsumerFilterManager());
        }
    }
    SelectMappedBufferResult result = consumeQueue.getIndexBuffer(requestHeader.getIndex());
    if (result == null) {
        response.setRemark(String.format("Index %d of %d@%s is not exist!", requestHeader.getIndex(), requestHeader.getQueueId(), requestHeader.getTopic()));
        return response;
    }
    try {
        List<ConsumeQueueData> queues = new ArrayList<>();
        for (int i = 0; i < result.getSize() && i < requestHeader.getCount() * ConsumeQueue.CQ_STORE_UNIT_SIZE; i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
            ConsumeQueueData one = new ConsumeQueueData();
            one.setPhysicOffset(result.getByteBuffer().getLong());
            one.setPhysicSize(result.getByteBuffer().getInt());
            one.setTagsCode(result.getByteBuffer().getLong());
            if (!consumeQueue.isExtAddr(one.getTagsCode())) {
                queues.add(one);
                continue;
            }
            ConsumeQueueExt.CqExtUnit cqExtUnit = consumeQueue.getExt(one.getTagsCode());
            if (cqExtUnit != null) {
                one.setExtendDataJson(JSON.toJSONString(cqExtUnit));
                if (cqExtUnit.getFilterBitMap() != null) {
                    one.setBitMap(BitsArray.create(cqExtUnit.getFilterBitMap()).toString());
                }
                if (messageFilter != null) {
                    one.setEval(messageFilter.isMatchedByConsumeQueue(cqExtUnit.getTagsCode(), cqExtUnit));
                }
            } else {
                one.setMsg("Cq extend not exist!addr: " + one.getTagsCode());
            }
            queues.add(one);
        }
        body.setQueueData(queues);
    } finally {
        result.release();
    }
    return response;
}

Example 45 with SubscriptionData

use of org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AdminBrokerProcessor method fetchAllConsumeStatsInBroker.

private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    GetConsumeStatsInBrokerHeader requestHeader = (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
    boolean isOrder = requestHeader.isOrder();
    ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroups = brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();
    List<Map<String, List<ConsumeStats>>> /* subscriptionGroupName */
    brokerConsumeStatsList = new ArrayList<Map<String, List<ConsumeStats>>>();
    long totalDiff = 0L;
    for (String group : subscriptionGroups.keySet()) {
        Map<String, List<ConsumeStats>> subscripTopicConsumeMap = new HashMap<String, List<ConsumeStats>>();
        Set<String> topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(group);
        List<ConsumeStats> consumeStatsList = new ArrayList<ConsumeStats>();
        for (String topic : topics) {
            ConsumeStats consumeStats = new ConsumeStats();
            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
            if (null == topicConfig) {
                log.warn("consumeStats, topic config not exist, {}", topic);
                continue;
            }
            if (isOrder && !topicConfig.isOrder()) {
                continue;
            }
            /**
             */
            {
                SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(group, topic);
                if (// 
                null == findSubscriptionData && this.brokerController.getConsumerManager().findSubscriptionDataCount(group) > 0) {
                    log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", group, topic);
                    continue;
                }
            }
            for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {
                MessageQueue mq = new MessageQueue();
                mq.setTopic(topic);
                mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
                mq.setQueueId(i);
                OffsetWrapper offsetWrapper = new OffsetWrapper();
                long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
                if (brokerOffset < 0)
                    brokerOffset = 0;
                long consumerOffset = // 
                this.brokerController.getConsumerOffsetManager().queryOffset(// 
                group, // 
                topic, i);
                if (consumerOffset < 0)
                    consumerOffset = 0;
                offsetWrapper.setBrokerOffset(brokerOffset);
                offsetWrapper.setConsumerOffset(consumerOffset);
                long timeOffset = consumerOffset - 1;
                if (timeOffset >= 0) {
                    long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
                    if (lastTimestamp > 0) {
                        offsetWrapper.setLastTimestamp(lastTimestamp);
                    }
                }
                consumeStats.getOffsetTable().put(mq, offsetWrapper);
            }
            double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(group, topic);
            consumeTps += consumeStats.getConsumeTps();
            consumeStats.setConsumeTps(consumeTps);
            totalDiff += consumeStats.computeTotalDiff();
            consumeStatsList.add(consumeStats);
        }
        subscripTopicConsumeMap.put(group, consumeStatsList);
        brokerConsumeStatsList.add(subscripTopicConsumeMap);
    }
    ConsumeStatsList consumeStats = new ConsumeStatsList();
    consumeStats.setBrokerAddr(brokerController.getBrokerAddr());
    consumeStats.setConsumeStatsList(brokerConsumeStatsList);
    consumeStats.setTotalDiff(totalDiff);
    response.setBody(consumeStats.encode());
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}