Examples with DefaultMessageStore org.apache.rocketmq.store.DefaultMessageStore used on opensource projects

Example 1 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class BrokerController method initialize.

public boolean initialize() throws CloneNotSupportedException {
    // 这几个load 加载 topics.json、 consumerOffset.json、 subscriptionGroup.json、 consumerFilter.json文件
    // 分别将各文件的数据存入 TopicConfigManager、 ConsumerOffsetManager、SubscriptionGroupManager  consumerFilterManager对象中
    boolean result = this.topicConfigManager.load();
    result = result && this.consumerOffsetManager.load();
    result = result && this.subscriptionGroupManager.load();
    result = result && this.consumerFilterManager.load();
    if (result) {
        try {
            // 初始化 DefaultMessageStore 对象，该对象是应用层访问存储层的访问类
            this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig);
            this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
            // load plugin
            MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
            this.messageStore = MessageStoreFactory.build(context, this.messageStore);
            this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));
        } catch (IOException e) {
            result = false;
            e.printStackTrace();
        }
    }
    // 加载数据
    result = result && this.messageStore.load();
    if (result) {
        // 初始化 Netty 服务端 NettyRemotingServer 对象
        this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
        NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
        fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2);
        // 定期检测客户端连接，清除不活动的连接
        this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
        // 初始化发送消息线程池
        this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getSendMessageThreadPoolNums(), this.brokerConfig.getSendMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.sendThreadPoolQueue, new ThreadFactoryImpl("SendMessageThread_"));
        // 初始化拉取消息线程池
        this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getPullMessageThreadPoolNums(), this.brokerConfig.getPullMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_"));
        // 初始化管理 Broker 线程池
        this.adminBrokerExecutor = Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), new ThreadFactoryImpl("AdminBrokerThread_"));
        // 初始化客户端管理线程池
        this.clientManageExecutor = new ThreadPoolExecutor(this.brokerConfig.getClientManageThreadPoolNums(), this.brokerConfig.getClientManageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientManagerThreadPoolQueue, new ThreadFactoryImpl("ClientManageThread_"));
        // 初始化消费端管理线程池
        this.consumerManageExecutor = Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl("ConsumerManageThread_"));
        // 注册事件处理器
        this.registerProcessor();
        // TODO remove in future
        final long initialDelay = UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis();
        final long period = 1000 * 60 * 60 * 24;
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.getBrokerStats().record();
                } catch (Throwable e) {
                    log.error("schedule record error.", e);
                }
            }
        }, initialDelay, period, TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.consumerOffsetManager.persist();
                } catch (Throwable e) {
                    log.error("schedule persist consumerOffset error.", e);
                }
            }
        }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.consumerFilterManager.persist();
                } catch (Throwable e) {
                    log.error("schedule persist consumer filter error.", e);
                }
            }
        }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.protectBroker();
                } catch (Exception e) {
                    log.error("protectBroker error.", e);
                }
            }
        }, 3, 3, TimeUnit.MINUTES);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.printWaterMark();
                } catch (Exception e) {
                    log.error("printWaterMark error.", e);
                }
            }
        }, 10, 1, TimeUnit.SECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    log.info("dispatch behind commit log {} bytes", BrokerController.this.getMessageStore().dispatchBehindBytes());
                } catch (Throwable e) {
                    log.error("schedule dispatchBehindBytes error.", e);
                }
            }
        }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        if (this.brokerConfig.getNamesrvAddr() != null) {
            this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());
            log.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
        } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.brokerOuterAPI.fetchNameServerAddr();
                    } catch (Throwable e) {
                        log.error("ScheduledTask fetchNameServerAddr exception", e);
                    }
                }
            }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
        }
        if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
            if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
                this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
                this.updateMasterHAServerAddrPeriodically = false;
            } else {
                this.updateMasterHAServerAddrPeriodically = true;
            }
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.slaveSynchronize.syncAll();
                    } catch (Throwable e) {
                        log.error("ScheduledTask syncAll slave exception", e);
                    }
                }
            }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        } else {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.printMasterAndSlaveDiff();
                    } catch (Throwable e) {
                        log.error("schedule printMasterAndSlaveDiff error.", e);
                    }
                }
            }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        }
    }
    return result;
}

Example 2 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AdminBrokerProcessor method ViewBrokerStatsData.

private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final ViewBrokerStatsDataRequestHeader requestHeader = (ViewBrokerStatsDataRequestHeader) request.decodeCommandCustomHeader(ViewBrokerStatsDataRequestHeader.class);
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    DefaultMessageStore messageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
    StatsItem statsItem = messageStore.getBrokerStatsManager().getStatsItem(requestHeader.getStatsName(), requestHeader.getStatsKey());
    if (null == statsItem) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark(String.format("The stats <%s> <%s> not exist", requestHeader.getStatsName(), requestHeader.getStatsKey()));
        return response;
    }
    BrokerStatsData brokerStatsData = new BrokerStatsData();
    {
        BrokerStatsItem it = new BrokerStatsItem();
        StatsSnapshot ss = statsItem.getStatsDataInMinute();
        it.setSum(ss.getSum());
        it.setTps(ss.getTps());
        it.setAvgpt(ss.getAvgpt());
        brokerStatsData.setStatsMinute(it);
    }
    {
        BrokerStatsItem it = new BrokerStatsItem();
        StatsSnapshot ss = statsItem.getStatsDataInHour();
        it.setSum(ss.getSum());
        it.setTps(ss.getTps());
        it.setAvgpt(ss.getAvgpt());
        brokerStatsData.setStatsHour(it);
    }
    {
        BrokerStatsItem it = new BrokerStatsItem();
        StatsSnapshot ss = statsItem.getStatsDataInDay();
        it.setSum(ss.getSum());
        it.setTps(ss.getTps());
        it.setAvgpt(ss.getAvgpt());
        brokerStatsData.setStatsDay(it);
    }
    response.setBody(brokerStatsData.encode());
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}

Example 3 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MessageStoreWithFilterTest method testGetMessage_withFilterBitMap.

@Test
public void testGetMessage_withFilterBitMap() {
    int topicCount = 10, msgPerTopic = 500;
    ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(topicCount, msgPerTopic);
    DefaultMessageStore master = null;
    try {
        master = gen(filterManager);
    } catch (Exception e) {
        e.printStackTrace();
        assertThat(true).isFalse();
    }
    try {
        List<MessageExtBrokerInner> msgs = null;
        try {
            msgs = putMsg(master, topicCount, msgPerTopic);
            // sleep to wait for consume queue has been constructed.
            Thread.sleep(1000);
        } catch (Exception e) {
            e.printStackTrace();
            assertThat(true).isFalse();
        }
        for (int i = 0; i < topicCount; i++) {
            String realTopic = topic + i;
            for (int j = 0; j < msgPerTopic; j++) {
                String group = "CID_" + j;
                ConsumerFilterData filterData = filterManager.get(realTopic, group);
                assertThat(filterData).isNotNull();
                List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, filterData);
                SubscriptionData subscriptionData = new SubscriptionData();
                subscriptionData.setExpressionType(filterData.getExpressionType());
                subscriptionData.setTopic(filterData.getTopic());
                subscriptionData.setClassFilterMode(false);
                subscriptionData.setSubString(filterData.getExpression());
                GetMessageResult getMessageResult = master.getMessage(group, realTopic, queueId, 0, 10000, new ExpressionMessageFilter(subscriptionData, filterData, filterManager));
                String assertMsg = group + "-" + realTopic;
                try {
                    assertThat(getMessageResult).isNotNull();
                    assertThat(GetMessageStatus.FOUND).isEqualTo(getMessageResult.getStatus());
                    assertThat(getMessageResult.getMessageBufferList()).isNotNull().isNotEmpty();
                    assertThat(getMessageResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
                    for (ByteBuffer buffer : getMessageResult.getMessageBufferList()) {
                        MessageExt messageExt = MessageDecoder.decode(buffer.slice(), false);
                        assertThat(messageExt).isNotNull();
                        Object evlRet = null;
                        try {
                            evlRet = filterData.getCompiledExpression().evaluate(new MessageEvaluationContext(messageExt.getProperties()));
                        } catch (Exception e) {
                            e.printStackTrace();
                            assertThat(true).isFalse();
                        }
                        assertThat(evlRet).isNotNull().isEqualTo(Boolean.TRUE);
                        // check
                        boolean find = false;
                        for (MessageExtBrokerInner messageExtBrokerInner : filteredMsgs) {
                            if (messageExtBrokerInner.getMsgId().equals(messageExt.getMsgId())) {
                                find = true;
                            }
                        }
                        assertThat(find).isTrue();
                    }
                } finally {
                    getMessageResult.release();
                }
            }
        }
    } finally {
        master.shutdown();
        master.destroy();
        deleteDirectory(storePath);
    }
}

Example 4 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq by apache.

the class BrokerController method initialize.

public boolean initialize() throws CloneNotSupportedException {
    boolean result = this.topicConfigManager.load();
    result = result && this.consumerOffsetManager.load();
    result = result && this.subscriptionGroupManager.load();
    result = result && this.consumerFilterManager.load();
    if (result) {
        try {
            this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig);
            this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
            // load plugin
            MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
            this.messageStore = MessageStoreFactory.build(context, this.messageStore);
            this.messageStore.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(this.brokerConfig, this.consumerFilterManager));
        } catch (IOException e) {
            result = false;
            log.error("Failed to initialize", e);
        }
    }
    result = result && this.messageStore.load();
    if (result) {
        this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
        NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();
        fastConfig.setListenPort(nettyServerConfig.getListenPort() - 2);
        this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
        this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getSendMessageThreadPoolNums(), this.brokerConfig.getSendMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.sendThreadPoolQueue, new ThreadFactoryImpl("SendMessageThread_"));
        this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getPullMessageThreadPoolNums(), this.brokerConfig.getPullMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_"));
        this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor(this.brokerConfig.getQueryMessageThreadPoolNums(), this.brokerConfig.getQueryMessageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.queryThreadPoolQueue, new ThreadFactoryImpl("QueryMessageThread_"));
        this.adminBrokerExecutor = Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), new ThreadFactoryImpl("AdminBrokerThread_"));
        this.clientManageExecutor = new ThreadPoolExecutor(this.brokerConfig.getClientManageThreadPoolNums(), this.brokerConfig.getClientManageThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientManagerThreadPoolQueue, new ThreadFactoryImpl("ClientManageThread_"));
        this.consumerManageExecutor = Executors.newFixedThreadPool(this.brokerConfig.getConsumerManageThreadPoolNums(), new ThreadFactoryImpl("ConsumerManageThread_"));
        this.registerProcessor();
        final long initialDelay = UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis();
        final long period = 1000 * 60 * 60 * 24;
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.getBrokerStats().record();
                } catch (Throwable e) {
                    log.error("schedule record error.", e);
                }
            }
        }, initialDelay, period, TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.consumerOffsetManager.persist();
                } catch (Throwable e) {
                    log.error("schedule persist consumerOffset error.", e);
                }
            }
        }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.consumerFilterManager.persist();
                } catch (Throwable e) {
                    log.error("schedule persist consumer filter error.", e);
                }
            }
        }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.protectBroker();
                } catch (Throwable e) {
                    log.error("protectBroker error.", e);
                }
            }
        }, 3, 3, TimeUnit.MINUTES);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    BrokerController.this.printWaterMark();
                } catch (Throwable e) {
                    log.error("printWaterMark error.", e);
                }
            }
        }, 10, 1, TimeUnit.SECONDS);
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                try {
                    log.info("dispatch behind commit log {} bytes", BrokerController.this.getMessageStore().dispatchBehindBytes());
                } catch (Throwable e) {
                    log.error("schedule dispatchBehindBytes error.", e);
                }
            }
        }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        if (this.brokerConfig.getNamesrvAddr() != null) {
            this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr());
            log.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr());
        } else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.brokerOuterAPI.fetchNameServerAddr();
                    } catch (Throwable e) {
                        log.error("ScheduledTask fetchNameServerAddr exception", e);
                    }
                }
            }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
        }
        if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
            if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
                this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
                this.updateMasterHAServerAddrPeriodically = false;
            } else {
                this.updateMasterHAServerAddrPeriodically = true;
            }
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.slaveSynchronize.syncAll();
                    } catch (Throwable e) {
                        log.error("ScheduledTask syncAll slave exception", e);
                    }
                }
            }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        } else {
            this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {

                @Override
                public void run() {
                    try {
                        BrokerController.this.printMasterAndSlaveDiff();
                    } catch (Throwable e) {
                        log.error("schedule printMasterAndSlaveDiff error.", e);
                    }
                }
            }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
        }
    }
    return result;
}

Example 5 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq by apache.

the class AdminBrokerProcessor method ViewBrokerStatsData.

private RemotingCommand ViewBrokerStatsData(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final ViewBrokerStatsDataRequestHeader requestHeader = (ViewBrokerStatsDataRequestHeader) request.decodeCommandCustomHeader(ViewBrokerStatsDataRequestHeader.class);
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    DefaultMessageStore messageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
    StatsItem statsItem = messageStore.getBrokerStatsManager().getStatsItem(requestHeader.getStatsName(), requestHeader.getStatsKey());
    if (null == statsItem) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark(String.format("The stats <%s> <%s> not exist", requestHeader.getStatsName(), requestHeader.getStatsKey()));
        return response;
    }
    BrokerStatsData brokerStatsData = new BrokerStatsData();
    {
        BrokerStatsItem it = new BrokerStatsItem();
        StatsSnapshot ss = statsItem.getStatsDataInMinute();
        it.setSum(ss.getSum());
        it.setTps(ss.getTps());
        it.setAvgpt(ss.getAvgpt());
        brokerStatsData.setStatsMinute(it);
    }
    {
        BrokerStatsItem it = new BrokerStatsItem();
        StatsSnapshot ss = statsItem.getStatsDataInHour();
        it.setSum(ss.getSum());
        it.setTps(ss.getTps());
        it.setAvgpt(ss.getAvgpt());
        brokerStatsData.setStatsHour(it);
    }
    {
        BrokerStatsItem it = new BrokerStatsItem();
        StatsSnapshot ss = statsItem.getStatsDataInDay();
        it.setSum(ss.getSum());
        it.setTps(ss.getTps());
        it.setAvgpt(ss.getAvgpt());
        brokerStatsData.setStatsDay(it);
    }
    response.setBody(brokerStatsData.encode());
    response.setCode(ResponseCode.SUCCESS);
    response.setRemark(null);
    return response;
}