Examples with DefaultMessageStore org.apache.rocketmq.store.DefaultMessageStore used on opensource projects

Example 6 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AdminBrokerProcessor method prepareRuntimeInfo.

// 准备运行时的broker的信息
private HashMap<String, String> prepareRuntimeInfo() {
    HashMap<String, String> runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo();
    runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));
    runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION));
    runtimeInfo.put("msgPutTotalYesterdayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalYesterdayMorning()));
    runtimeInfo.put("msgPutTotalTodayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayMorning()));
    runtimeInfo.put("msgPutTotalTodayNow", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayNow()));
    runtimeInfo.put("msgGetTotalYesterdayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalYesterdayMorning()));
    runtimeInfo.put("msgGetTotalTodayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayMorning()));
    runtimeInfo.put("msgGetTotalTodayNow", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayNow()));
    runtimeInfo.put("sendThreadPoolQueueSize", String.valueOf(this.brokerController.getSendThreadPoolQueue().size()));
    runtimeInfo.put("sendThreadPoolQueueCapacity", String.valueOf(this.brokerController.getBrokerConfig().getSendThreadPoolQueueCapacity()));
    runtimeInfo.put("pullThreadPoolQueueSize", String.valueOf(this.brokerController.getPullThreadPoolQueue().size()));
    runtimeInfo.put("pullThreadPoolQueueCapacity", String.valueOf(this.brokerController.getBrokerConfig().getPullThreadPoolQueueCapacity()));
    runtimeInfo.put("dispatchBehindBytes", String.valueOf(this.brokerController.getMessageStore().dispatchBehindBytes()));
    runtimeInfo.put("pageCacheLockTimeMills", String.valueOf(this.brokerController.getMessageStore().lockTimeMills()));
    runtimeInfo.put("sendThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4SendThreadPoolQueue()));
    runtimeInfo.put("pullThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4PullThreadPoolQueue()));
    runtimeInfo.put("earliestMessageTimeStamp", String.valueOf(this.brokerController.getMessageStore().getEarliestMessageTime()));
    runtimeInfo.put("startAcceptSendRequestTimeStamp", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp()));
    if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) {
        DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
        runtimeInfo.put("remainTransientStoreBufferNumbs", String.valueOf(defaultMessageStore.remainTransientStoreBufferNumbs()));
        if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            runtimeInfo.put("remainHowManyDataToCommit", MixAll.humanReadableByteCount(defaultMessageStore.getCommitLog().remainHowManyDataToCommit(), false));
        }
        runtimeInfo.put("remainHowManyDataToFlush", MixAll.humanReadableByteCount(defaultMessageStore.getCommitLog().remainHowManyDataToFlush(), false));
    }
    java.io.File commitLogDir = new java.io.File(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
    if (commitLogDir.exists()) {
        runtimeInfo.put("commitLogDirCapacity", String.format("Total : %s, Free : %s.", MixAll.humanReadableByteCount(commitLogDir.getTotalSpace(), false), MixAll.humanReadableByteCount(commitLogDir.getFreeSpace(), false)));
    }
    return runtimeInfo;
}

Example 7 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MessageStoreWithFilterTest method testGetMessage_withFilterBitMapAndConsumerChanged.

@Test
public void testGetMessage_withFilterBitMapAndConsumerChanged() {
    int topicCount = 10, msgPerTopic = 10;
    ConsumerFilterManager filterManager = ConsumerFilterManagerTest.gen(topicCount, msgPerTopic);
    DefaultMessageStore master = null;
    try {
        master = gen(filterManager);
    } catch (Exception e) {
        e.printStackTrace();
        assertThat(true).isFalse();
    }
    try {
        List<MessageExtBrokerInner> msgs = null;
        try {
            msgs = putMsg(master, topicCount, msgPerTopic);
        } catch (Exception e) {
            e.printStackTrace();
            assertThat(true).isFalse();
        }
        // sleep to wait for consume queue has been constructed.
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
            assertThat(true).isFalse();
        }
        // reset consumer;
        String topic = "topic" + 0;
        String resetGroup = "CID_" + 2;
        String normalGroup = "CID_" + 3;
        {
            // reset CID_2@topic0 to get all messages.
            SubscriptionData resetSubData = new SubscriptionData();
            resetSubData.setExpressionType(ExpressionType.SQL92);
            resetSubData.setTopic(topic);
            resetSubData.setClassFilterMode(false);
            resetSubData.setSubString("a is not null OR a is null");
            ConsumerFilterData resetFilterData = ConsumerFilterManager.build(topic, resetGroup, resetSubData.getSubString(), resetSubData.getExpressionType(), System.currentTimeMillis());
            GetMessageResult resetGetResult = master.getMessage(resetGroup, topic, queueId, 0, 1000, new ExpressionMessageFilter(resetSubData, resetFilterData, filterManager));
            try {
                assertThat(resetGetResult).isNotNull();
                List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, resetFilterData);
                assertThat(resetGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
            } finally {
                resetGetResult.release();
            }
        }
        {
            ConsumerFilterData normalFilterData = filterManager.get(topic, normalGroup);
            assertThat(normalFilterData).isNotNull();
            assertThat(normalFilterData.getBornTime()).isLessThan(System.currentTimeMillis());
            SubscriptionData normalSubData = new SubscriptionData();
            normalSubData.setExpressionType(normalFilterData.getExpressionType());
            normalSubData.setTopic(topic);
            normalSubData.setClassFilterMode(false);
            normalSubData.setSubString(normalFilterData.getExpression());
            List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, normalFilterData);
            GetMessageResult normalGetResult = master.getMessage(normalGroup, topic, queueId, 0, 1000, new ExpressionMessageFilter(normalSubData, normalFilterData, filterManager));
            try {
                assertThat(normalGetResult).isNotNull();
                assertThat(normalGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
            } finally {
                normalGetResult.release();
            }
        }
    } finally {
        master.shutdown();
        master.destroy();
        deleteDirectory(storePath);
    }
}

Example 8 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MessageStoreWithFilterTest method gen.

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(commitLogFileSize, cqFileSize, true, cqExtFileSize);
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);
    DefaultMessageStore master = new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager(brokerConfig.getBrokerClusterName()), new MessageArrivingListener() {

        @Override
        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
        // System.out.println(String.format("Msg coming: %s, %d, %d, %d",
        // topic, queueId, logicOffset, tagsCode));
        }
    }, brokerConfig);
    master.getDispatcherList().addFirst(new CommitLogDispatcher() {

        @Override
        public void dispatch(DispatchRequest request) {
            try {
            // System.out.println(String.format("offset:%d, bitMap:%s", request.getCommitLogOffset(),
            // BitsArray.create(request.getBitMap()).toString()));
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));
    assertThat(master.load()).isTrue();
    master.start();
    return master;
}

Example 9 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq by apache.

the class AdminBrokerProcessor method prepareRuntimeInfo.

private HashMap<String, String> prepareRuntimeInfo() {
    HashMap<String, String> runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo();
    runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));
    runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION));
    runtimeInfo.put("msgPutTotalYesterdayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalYesterdayMorning()));
    runtimeInfo.put("msgPutTotalTodayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayMorning()));
    runtimeInfo.put("msgPutTotalTodayNow", String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayNow()));
    runtimeInfo.put("msgGetTotalYesterdayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalYesterdayMorning()));
    runtimeInfo.put("msgGetTotalTodayMorning", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayMorning()));
    runtimeInfo.put("msgGetTotalTodayNow", String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayNow()));
    runtimeInfo.put("sendThreadPoolQueueSize", String.valueOf(this.brokerController.getSendThreadPoolQueue().size()));
    runtimeInfo.put("sendThreadPoolQueueCapacity", String.valueOf(this.brokerController.getBrokerConfig().getSendThreadPoolQueueCapacity()));
    runtimeInfo.put("pullThreadPoolQueueSize", String.valueOf(this.brokerController.getPullThreadPoolQueue().size()));
    runtimeInfo.put("pullThreadPoolQueueCapacity", String.valueOf(this.brokerController.getBrokerConfig().getPullThreadPoolQueueCapacity()));
    runtimeInfo.put("queryThreadPoolQueueSize", String.valueOf(this.brokerController.getQueryThreadPoolQueue().size()));
    runtimeInfo.put("queryThreadPoolQueueCapacity", String.valueOf(this.brokerController.getBrokerConfig().getQueryThreadPoolQueueCapacity()));
    runtimeInfo.put("dispatchBehindBytes", String.valueOf(this.brokerController.getMessageStore().dispatchBehindBytes()));
    runtimeInfo.put("pageCacheLockTimeMills", String.valueOf(this.brokerController.getMessageStore().lockTimeMills()));
    runtimeInfo.put("sendThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4SendThreadPoolQueue()));
    runtimeInfo.put("pullThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4PullThreadPoolQueue()));
    runtimeInfo.put("queryThreadPoolQueueHeadWaitTimeMills", String.valueOf(this.brokerController.headSlowTimeMills4QueryThreadPoolQueue()));
    runtimeInfo.put("earliestMessageTimeStamp", String.valueOf(this.brokerController.getMessageStore().getEarliestMessageTime()));
    runtimeInfo.put("startAcceptSendRequestTimeStamp", String.valueOf(this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp()));
    if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) {
        DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
        runtimeInfo.put("remainTransientStoreBufferNumbs", String.valueOf(defaultMessageStore.remainTransientStoreBufferNumbs()));
        if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) {
            runtimeInfo.put("remainHowManyDataToCommit", MixAll.humanReadableByteCount(defaultMessageStore.getCommitLog().remainHowManyDataToCommit(), false));
        }
        runtimeInfo.put("remainHowManyDataToFlush", MixAll.humanReadableByteCount(defaultMessageStore.getCommitLog().remainHowManyDataToFlush(), false));
    }
    java.io.File commitLogDir = new java.io.File(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
    if (commitLogDir.exists()) {
        runtimeInfo.put("commitLogDirCapacity", String.format("Total : %s, Free : %s.", MixAll.humanReadableByteCount(commitLogDir.getTotalSpace(), false), MixAll.humanReadableByteCount(commitLogDir.getFreeSpace(), false)));
    }
    return runtimeInfo;
}

Example 10 with DefaultMessageStore

use of org.apache.rocketmq.store.DefaultMessageStore in project rocketmq by apache.

the class MessageStoreWithFilterTest method gen.

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(commitLogFileSize, cqFileSize, true, cqExtFileSize);
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);
    DefaultMessageStore master = new DefaultMessageStore(messageStoreConfig, new BrokerStatsManager(brokerConfig.getBrokerClusterName()), new MessageArrivingListener() {

        @Override
        public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
        }
    }, brokerConfig);
    master.getDispatcherList().addFirst(new CommitLogDispatcher() {

        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));
    assertThat(master.load()).isTrue();
    master.start();
    return master;
}