Examples with SubscriptionGroupConfig org.apache.rocketmq.common.subscription.SubscriptionGroupConfig used on opensource projects

Example 26 with SubscriptionGroupConfig

use of org.apache.rocketmq.common.subscription.SubscriptionGroupConfig in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class DefaultMQAdminExtTest method init.

@BeforeClass
public static void init() throws Exception {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);
    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);
    properties.setProperty("maxMessageSize", "5000000");
    properties.setProperty("flushDelayOffsetInterval", "15000");
    properties.setProperty("serverSocketRcvBufSize", "655350");
    when(mQClientAPIImpl.getBrokerConfig(anyString(), anyLong())).thenReturn(properties);
    Set<String> topicSet = new HashSet<>();
    topicSet.add("topic_one");
    topicSet.add("topic_two");
    topicList.setTopicList(topicSet);
    when(mQClientAPIImpl.getTopicListFromNameServer(anyLong())).thenReturn(topicList);
    List<BrokerData> brokerDatas = new ArrayList<>();
    HashMap<Long, String> brokerAddrs = new HashMap<>();
    brokerAddrs.put(1234l, "127.0.0.1:10911");
    BrokerData brokerData = new BrokerData();
    brokerData.setCluster("default-cluster");
    brokerData.setBrokerName("default-broker");
    brokerData.setBrokerAddrs(brokerAddrs);
    brokerDatas.add(brokerData);
    topicRouteData.setBrokerDatas(brokerDatas);
    topicRouteData.setQueueDatas(new ArrayList<QueueData>());
    topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    HashMap<String, String> result = new HashMap<>();
    result.put("id", "1234");
    result.put("brokerName", "default-broker");
    kvTable.setTable(result);
    when(mQClientAPIImpl.getBrokerRuntimeInfo(anyString(), anyLong())).thenReturn(kvTable);
    HashMap<String, BrokerData> brokerAddrTable = new HashMap<>();
    brokerAddrTable.put("default-broker", brokerData);
    brokerAddrTable.put("broker-test", new BrokerData());
    clusterInfo.setBrokerAddrTable(brokerAddrTable);
    clusterInfo.setClusterAddrTable(new HashMap<String, Set<String>>());
    when(mQClientAPIImpl.getBrokerClusterInfo(anyLong())).thenReturn(clusterInfo);
    when(mQClientAPIImpl.cleanExpiredConsumeQueue(anyString(), anyLong())).thenReturn(true);
    Set<String> clusterList = new HashSet<>();
    clusterList.add("default-cluster-one");
    clusterList.add("default-cluster-two");
    when(mQClientAPIImpl.getClusterList(anyString(), anyLong())).thenReturn(clusterList);
    GroupList groupList = new GroupList();
    HashSet<String> groups = new HashSet<>();
    groups.add("consumer-group-one");
    groups.add("consumer-group-two");
    groupList.setGroupList(groups);
    when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData);
    when(mQClientAPIImpl.queryTopicConsumeByWho(anyString(), anyString(), anyLong())).thenReturn(groupList);
    SubscriptionGroupWrapper subscriptionGroupWrapper = new SubscriptionGroupWrapper();
    ConcurrentHashMap<String, SubscriptionGroupConfig> subscriptions = new ConcurrentHashMap<>();
    SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
    subscriptionGroupConfig.setConsumeBroadcastEnable(true);
    subscriptionGroupConfig.setBrokerId(1234);
    subscriptionGroupConfig.setGroupName("Consumer-group-one");
    subscriptions.put("Consumer-group-one", subscriptionGroupConfig);
    subscriptionGroupWrapper.setSubscriptionGroupTable(subscriptions);
    when(mQClientAPIImpl.getAllSubscriptionGroup(anyString(), anyLong())).thenReturn(subscriptionGroupWrapper);
    String topicListConfig = "topicListConfig";
    when(mQClientAPIImpl.getKVConfigValue(anyString(), anyString(), anyLong())).thenReturn(topicListConfig);
    KVTable kvTable = new KVTable();
    HashMap<String, String> kv = new HashMap<>();
    kv.put("broker-name", "broker-one");
    kv.put("cluster-name", "default-cluster");
    kvTable.setTable(kv);
    when(mQClientAPIImpl.getKVListByNamespace(anyString(), anyLong())).thenReturn(kvTable);
    ConsumeStats consumeStats = new ConsumeStats();
    consumeStats.setConsumeTps(1234);
    MessageQueue messageQueue = new MessageQueue();
    OffsetWrapper offsetWrapper = new OffsetWrapper();
    HashMap<MessageQueue, OffsetWrapper> stats = new HashMap<>();
    stats.put(messageQueue, offsetWrapper);
    consumeStats.setOffsetTable(stats);
    when(mQClientAPIImpl.getConsumeStats(anyString(), anyString(), anyString(), anyLong())).thenReturn(consumeStats);
    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    connections.add(new Connection());
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
    ProducerConnection producerConnection = new ProducerConnection();
    Connection connection = new Connection();
    connection.setClientAddr("127.0.0.1:9898");
    connection.setClientId("PID_12345");
    HashSet<Connection> connectionSet = new HashSet<Connection>();
    connectionSet.add(connection);
    producerConnection.setConnectionSet(connectionSet);
    when(mQClientAPIImpl.getProducerConnectionList(anyString(), anyString(), anyLong())).thenReturn(producerConnection);
    when(mQClientAPIImpl.wipeWritePermOfBroker(anyString(), anyString(), anyLong())).thenReturn(6);
    TopicStatsTable topicStatsTable = new TopicStatsTable();
    topicStatsTable.setOffsetTable(new HashMap<MessageQueue, TopicOffset>());
    Map<String, Map<MessageQueue, Long>> consumerStatus = new HashMap<>();
    when(mQClientAPIImpl.invokeBrokerToGetConsumerStatus(anyString(), anyString(), anyString(), anyString(), anyLong())).thenReturn(consumerStatus);
    List<QueueTimeSpan> queueTimeSpanList = new ArrayList<>();
    when(mQClientAPIImpl.queryConsumeTimeSpan(anyString(), anyString(), anyString(), anyLong())).thenReturn(queueTimeSpanList);
    ConsumerRunningInfo consumerRunningInfo = new ConsumerRunningInfo();
    consumerRunningInfo.setJstack("test");
    consumerRunningInfo.setMqTable(new TreeMap<MessageQueue, ProcessQueueInfo>());
    consumerRunningInfo.setStatusTable(new TreeMap<String, ConsumeStatus>());
    consumerRunningInfo.setSubscriptionSet(new TreeSet<SubscriptionData>());
    when(mQClientAPIImpl.getConsumerRunningInfo(anyString(), anyString(), anyString(), anyBoolean(), anyLong())).thenReturn(consumerRunningInfo);
}

Example 27 with SubscriptionGroupConfig

use of org.apache.rocketmq.common.subscription.SubscriptionGroupConfig in project rocketmq by apache.

the class UpdateSubGroupSubCommand method execute.

@Override
public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) throws SubCommandException {
    DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
    defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
    try {
        SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
        subscriptionGroupConfig.setConsumeBroadcastEnable(false);
        subscriptionGroupConfig.setConsumeFromMinEnable(false);
        // groupName
        subscriptionGroupConfig.setGroupName(commandLine.getOptionValue('g').trim());
        // consumeEnable
        if (commandLine.hasOption('s')) {
            subscriptionGroupConfig.setConsumeEnable(Boolean.parseBoolean(commandLine.getOptionValue('s').trim()));
        }
        // consumeFromMinEnable
        if (commandLine.hasOption('m')) {
            subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.parseBoolean(commandLine.getOptionValue('m').trim()));
        }
        // consumeBroadcastEnable
        if (commandLine.hasOption('d')) {
            subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.parseBoolean(commandLine.getOptionValue('d').trim()));
        }
        // retryQueueNums
        if (commandLine.hasOption('q')) {
            subscriptionGroupConfig.setRetryQueueNums(Integer.parseInt(commandLine.getOptionValue('q').trim()));
        }
        // retryMaxTimes
        if (commandLine.hasOption('r')) {
            subscriptionGroupConfig.setRetryMaxTimes(Integer.parseInt(commandLine.getOptionValue('r').trim()));
        }
        // brokerId
        if (commandLine.hasOption('i')) {
            subscriptionGroupConfig.setBrokerId(Long.parseLong(commandLine.getOptionValue('i').trim()));
        }
        // whichBrokerWhenConsumeSlowly
        if (commandLine.hasOption('w')) {
            subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(Long.parseLong(commandLine.getOptionValue('w').trim()));
        }
        // notifyConsumerIdsChanged
        if (commandLine.hasOption('a')) {
            subscriptionGroupConfig.setNotifyConsumerIdsChangedEnable(Boolean.parseBoolean(commandLine.getOptionValue('a').trim()));
        }
        if (commandLine.hasOption('b')) {
            String addr = commandLine.getOptionValue('b').trim();
            defaultMQAdminExt.start();
            defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subscriptionGroupConfig);
            System.out.printf("create subscription group to %s success.%n", addr);
            System.out.printf("%s", subscriptionGroupConfig);
            return;
        } else if (commandLine.hasOption('c')) {
            String clusterName = commandLine.getOptionValue('c').trim();
            defaultMQAdminExt.start();
            Set<String> masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
            for (String addr : masterSet) {
                try {
                    defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subscriptionGroupConfig);
                    System.out.printf("create subscription group to %s success.%n", addr);
                } catch (Exception e) {
                    e.printStackTrace();
                    Thread.sleep(1000 * 1);
                }
            }
            System.out.printf("%s", subscriptionGroupConfig);
            return;
        }
        ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options);
    } catch (Exception e) {
        throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e);
    } finally {
        defaultMQAdminExt.shutdown();
    }
}

Example 28 with SubscriptionGroupConfig

use of org.apache.rocketmq.common.subscription.SubscriptionGroupConfig in project rocketmq by apache.

the class AdminBrokerProcessor method fetchAllConsumeStatsInBroker.

private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    GetConsumeStatsInBrokerHeader requestHeader = (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
    boolean isOrder = requestHeader.isOrder();
    ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroups = brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();
    List<Map<String, List<ConsumeStats>>> /* subscriptionGroupName */
    brokerConsumeStatsList = new ArrayList<Map<String, List<ConsumeStats>>>();
    long totalDiff = 0L;
    for (String group : subscriptionGroups.keySet()) {
        Map<String, List<ConsumeStats>> subscripTopicConsumeMap = new HashMap<String, List<ConsumeStats>>();
        Set<String> topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(group);
        List<ConsumeStats> consumeStatsList = new ArrayList<ConsumeStats>();
        for (String topic : topics) {
            ConsumeStats consumeStats = new ConsumeStats();
            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
            if (null == topicConfig) {
                log.warn("consumeStats, topic config not exist, {}", topic);
                continue;
            }
            if (isOrder && !topicConfig.isOrder()) {
                continue;
            }
            {
                SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(group, topic);
                if (null == findSubscriptionData && this.brokerController.getConsumerManager().findSubscriptionDataCount(group) > 0) {
                    log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", group, topic);
                    continue;
                }
            }
            for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {
                MessageQueue mq = new MessageQueue();
                mq.setTopic(topic);
                mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
                mq.setQueueId(i);
                OffsetWrapper offsetWrapper = new OffsetWrapper();
                long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
                if (brokerOffset < 0)
                    brokerOffset = 0;
                long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, i);
                if (consumerOffset < 0)
                    consumerOffset = 0;
                offsetWrapper.setBrokerOffset(brokerOffset);
                offsetWrapper.setConsumerOffset(consumerOffset);
                long timeOffset = consumerOffset - 1;
                if (timeOffset >= 0) {
                    long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
                    if (lastTimestamp > 0) {
                        offsetWrapper.setLastTimestamp(lastTimestamp);
                    }
                }
                consumeStats.getOffsetTable().put(mq, offsetWrapper);
            }
            double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(group, topic);
            consumeTps += consumeStats.getConsumeTps();
            consumeStats.setConsumeTps(consumeTps);
            totalDiff += consumeStats.computeTotalDiff();
            consumeStatsList.add(consumeStats);
        }
        subscripTopicConsumeMap.put(group, consumeStatsList);
        brokerConsumeStatsList.add(subscripTopicConsumeMap);
    }
    ConsumeStatsList consumeStats = new ConsumeStatsList();
    consumeStats.setBrokerAddr(brokerController.getBrokerAddr());
    consumeStats.setConsumeStatsList(brokerConsumeStatsList);
    consumeStats.setTotalDiff(totalDiff);
    response.setBody(consumeStats.encode());
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}

Example 29 with SubscriptionGroupConfig

use of org.apache.rocketmq.common.subscription.SubscriptionGroupConfig in project rocketmq by apache.

the class ClientManageProcessor method heartBeat.

public RemotingCommand heartBeat(ChannelHandlerContext ctx, RemotingCommand request) {
    RemotingCommand response = RemotingCommand.createResponseCommand(null);
    HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);
    ClientChannelInfo clientChannelInfo = new ClientChannelInfo(ctx.channel(), heartbeatData.getClientID(), request.getLanguage(), request.getVersion());
    for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(data.getGroupName());
        boolean isNotifyConsumerIdsChangedEnable = true;
        if (null != subscriptionGroupConfig) {
            isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable();
            int topicSysFlag = 0;
            if (data.isUnitMode()) {
                topicSysFlag = TopicSysFlag.buildSysFlag(false, true);
            }
            String newTopic = MixAll.getRetryTopic(data.getGroupName());
            this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic, subscriptionGroupConfig.getRetryQueueNums(), PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag);
        }
        boolean changed = this.brokerController.getConsumerManager().registerConsumer(data.getGroupName(), clientChannelInfo, data.getConsumeType(), data.getMessageModel(), data.getConsumeFromWhere(), data.getSubscriptionDataSet(), isNotifyConsumerIdsChangedEnable);
        if (changed) {
            log.info("registerConsumer info changed {} {}", data.toString(), RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
        }
    }
    for (ProducerData data : heartbeatData.getProducerDataSet()) {
        this.brokerController.getProducerManager().registerProducer(data.getGroupName(), clientChannelInfo);
    }
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}

Example 30 with SubscriptionGroupConfig

use of org.apache.rocketmq.common.subscription.SubscriptionGroupConfig in project rocketmq by apache.

the class SendMessageProcessor method handleRetryAndDLQ.

private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response, RemotingCommand request, MessageExt msg, TopicConfig topicConfig) {
    String newTopic = requestHeader.getTopic();
    if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
        String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());
        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(groupName);
        if (null == subscriptionGroupConfig) {
            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
            response.setRemark("subscription group not exist, " + groupName + " " + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));
            return false;
        }
        int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();
        if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {
            maxReconsumeTimes = requestHeader.getMaxReconsumeTimes();
        }
        int reconsumeTimes = requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes();
        if (reconsumeTimes >= maxReconsumeTimes) {
            newTopic = MixAll.getDLQTopic(groupName);
            int queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP;
            topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic, DLQ_NUMS_PER_GROUP, PermName.PERM_WRITE, 0);
            msg.setTopic(newTopic);
            msg.setQueueId(queueIdInt);
            if (null == topicConfig) {
                response.setCode(ResponseCode.SYSTEM_ERROR);
                response.setRemark("topic[" + newTopic + "] not exist");
                return false;
            }
        }
    }
    int sysFlag = requestHeader.getSysFlag();
    if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) {
        sysFlag |= MessageSysFlag.MULTI_TAGS_FLAG;
    }
    msg.setSysFlag(sysFlag);
    return true;
}