Examples with MessageExtBrokerInner org.apache.rocketmq.store.MessageExtBrokerInner used on opensource projects

Example 11 with MessageExtBrokerInner

use of org.apache.rocketmq.store.MessageExtBrokerInner in project rocketmq by apache.

the class MessageStoreWithFilterTest method testGetMessage_withFilterBitMapAndConsumerChanged.

@Test
public void testGetMessage_withFilterBitMapAndConsumerChanged() throws Exception {
    List<MessageExtBrokerInner> msgs = putMsg(master, topicCount, msgPerTopic);
    Thread.sleep(200);
    // reset consumer;
    String topic = "topic" + 0;
    String resetGroup = "CID_" + 2;
    String normalGroup = "CID_" + 3;
    {
        // reset CID_2@topic0 to get all messages.
        SubscriptionData resetSubData = new SubscriptionData();
        resetSubData.setExpressionType(ExpressionType.SQL92);
        resetSubData.setTopic(topic);
        resetSubData.setClassFilterMode(false);
        resetSubData.setSubString("a is not null OR a is null");
        ConsumerFilterData resetFilterData = ConsumerFilterManager.build(topic, resetGroup, resetSubData.getSubString(), resetSubData.getExpressionType(), System.currentTimeMillis());
        GetMessageResult resetGetResult = master.getMessage(resetGroup, topic, queueId, 0, 1000, new ExpressionMessageFilter(resetSubData, resetFilterData, filterManager));
        try {
            assertThat(resetGetResult).isNotNull();
            List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, resetFilterData);
            assertThat(resetGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
        } finally {
            resetGetResult.release();
        }
    }
    {
        ConsumerFilterData normalFilterData = filterManager.get(topic, normalGroup);
        assertThat(normalFilterData).isNotNull();
        assertThat(normalFilterData.getBornTime()).isLessThan(System.currentTimeMillis());
        SubscriptionData normalSubData = new SubscriptionData();
        normalSubData.setExpressionType(normalFilterData.getExpressionType());
        normalSubData.setTopic(topic);
        normalSubData.setClassFilterMode(false);
        normalSubData.setSubString(normalFilterData.getExpression());
        List<MessageExtBrokerInner> filteredMsgs = filtered(msgs, normalFilterData);
        GetMessageResult normalGetResult = master.getMessage(normalGroup, topic, queueId, 0, 1000, new ExpressionMessageFilter(normalSubData, normalFilterData, filterManager));
        try {
            assertThat(normalGetResult).isNotNull();
            assertThat(normalGetResult.getMessageBufferList().size()).isEqualTo(filteredMsgs.size());
        } finally {
            normalGetResult.release();
        }
    }
}

Example 12 with MessageExtBrokerInner

use of org.apache.rocketmq.store.MessageExtBrokerInner in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class EndTransactionProcessor method processRequest.

@Override
public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException {
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    final EndTransactionRequestHeader requestHeader = (EndTransactionRequestHeader) request.decodeCommandCustomHeader(EndTransactionRequestHeader.class);
    if (requestHeader.getFromTransactionCheck()) {
        switch(requestHeader.getCommitOrRollback()) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
                {
                    LOGGER.warn("check producer[{}] transaction state, but it's pending status." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), request.getRemark());
                    return null;
                }
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                {
                    LOGGER.warn("check producer[{}] transaction state, the producer commit the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), request.getRemark());
                    break;
                }
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                {
                    LOGGER.warn("check producer[{}] transaction state, the producer rollback the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), request.getRemark());
                    break;
                }
            default:
                return null;
        }
    } else {
        switch(requestHeader.getCommitOrRollback()) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
                {
                    LOGGER.warn("the producer[{}] end transaction in sending message,  and it's pending status." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), request.getRemark());
                    return null;
                }
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                {
                    break;
                }
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                {
                    LOGGER.warn("the producer[{}] end transaction in sending message, rollback the message." + "RequestHeader: {} Remark: {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.toString(), request.getRemark());
                    break;
                }
            default:
                return null;
        }
    }
    final MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getCommitLogOffset());
    if (msgExt != null) {
        final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP);
        if (!pgroupRead.equals(requestHeader.getProducerGroup())) {
            response.setCode(ResponseCode.SYSTEM_ERROR);
            response.setRemark("the producer group wrong");
            return response;
        }
        if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) {
            response.setCode(ResponseCode.SYSTEM_ERROR);
            response.setRemark("the transaction state table offset wrong");
            return response;
        }
        if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) {
            response.setCode(ResponseCode.SYSTEM_ERROR);
            response.setRemark("the commit log offset wrong");
            return response;
        }
        MessageExtBrokerInner msgInner = this.endMessageTransaction(msgExt);
        msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), requestHeader.getCommitOrRollback()));
        msgInner.setQueueOffset(requestHeader.getTranStateTableOffset());
        msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset());
        msgInner.setStoreTimestamp(msgExt.getStoreTimestamp());
        if (MessageSysFlag.TRANSACTION_ROLLBACK_TYPE == requestHeader.getCommitOrRollback()) {
            msgInner.setBody(null);
        }
        final MessageStore messageStore = this.brokerController.getMessageStore();
        final PutMessageResult putMessageResult = messageStore.putMessage(msgInner);
        if (putMessageResult != null) {
            switch(putMessageResult.getPutMessageStatus()) {
                // Success
                case PUT_OK:
                case FLUSH_DISK_TIMEOUT:
                case FLUSH_SLAVE_TIMEOUT:
                case SLAVE_NOT_AVAILABLE:
                    response.setCode(ResponseCode.SUCCESS);
                    response.setRemark(null);
                    break;
                // Failed
                case CREATE_MAPEDFILE_FAILED:
                    response.setCode(ResponseCode.SYSTEM_ERROR);
                    response.setRemark("create mapped file failed.");
                    break;
                case MESSAGE_ILLEGAL:
                case PROPERTIES_SIZE_EXCEEDED:
                    response.setCode(ResponseCode.MESSAGE_ILLEGAL);
                    response.setRemark("the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k.");
                    break;
                case SERVICE_NOT_AVAILABLE:
                    response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
                    response.setRemark("service not available now.");
                    break;
                case OS_PAGECACHE_BUSY:
                    response.setCode(ResponseCode.SYSTEM_ERROR);
                    response.setRemark("OS page cache busy, please try another machine");
                    break;
                case UNKNOWN_ERROR:
                    response.setCode(ResponseCode.SYSTEM_ERROR);
                    response.setRemark("UNKNOWN_ERROR");
                    break;
                default:
                    response.setCode(ResponseCode.SYSTEM_ERROR);
                    response.setRemark("UNKNOWN_ERROR DEFAULT");
                    break;
            }
            return response;
        } else {
            response.setCode(ResponseCode.SYSTEM_ERROR);
            response.setRemark("store putMessage return null");
        }
    } else {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark("find prepared transaction message failed");
        return response;
    }
    return response;
}

Example 13 with MessageExtBrokerInner

use of org.apache.rocketmq.store.MessageExtBrokerInner in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class EndTransactionProcessor method endMessageTransaction.

private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) {
    MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
    msgInner.setBody(msgExt.getBody());
    msgInner.setFlag(msgExt.getFlag());
    MessageAccessor.setProperties(msgInner, msgExt.getProperties());
    TopicFilterType topicFilterType = (msgInner.getSysFlag() & MessageSysFlag.MULTI_TAGS_FLAG) == MessageSysFlag.MULTI_TAGS_FLAG ? TopicFilterType.MULTI_TAG : TopicFilterType.SINGLE_TAG;
    long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags());
    msgInner.setTagsCode(tagsCodeValue);
    msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
    msgInner.setSysFlag(msgExt.getSysFlag());
    msgInner.setBornTimestamp(msgExt.getBornTimestamp());
    msgInner.setBornHost(msgExt.getBornHost());
    msgInner.setStoreHost(msgExt.getStoreHost());
    msgInner.setReconsumeTimes(msgExt.getReconsumeTimes());
    msgInner.setWaitStoreMsgOK(false);
    MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL);
    msgInner.setTopic(msgExt.getTopic());
    msgInner.setQueueId(msgExt.getQueueId());
    return msgInner;
}

Example 14 with MessageExtBrokerInner

use of org.apache.rocketmq.store.MessageExtBrokerInner in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class SendMessageProcessor method consumerSendMsgBack.

// Broker Consumer将处理不了的消息发回服务器
private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, final RemotingCommand request) throws RemotingCommandException {
    final RemotingCommand response = RemotingCommand.createResponseCommand(null);
    final ConsumerSendMsgBackRequestHeader requestHeader = (ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class);
    // 消息轨迹：记录消费失败的消息
    if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) {
        // 执行hook
        ConsumeMessageContext context = new ConsumeMessageContext();
        context.setConsumerGroup(requestHeader.getGroup());
        context.setTopic(requestHeader.getOriginTopic());
        context.setCommercialRcvStats(BrokerStatsManager.StatsType.SEND_BACK);
        context.setCommercialRcvTimes(1);
        context.setCommercialOwner(request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER));
        this.executeConsumeMessageHookAfter(context);
    }
    // 确保订阅组存在
    SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup());
    if (null == subscriptionGroupConfig) {
        response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
        response.setRemark("subscription group not exist, " + requestHeader.getGroup() + " " + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));
        return response;
    }
    // 检查Broker权限
    if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
        response.setCode(ResponseCode.NO_PERMISSION);
        response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending message is forbidden");
        return response;
    }
    // 如果重试队列数目为0，则直接丢弃消息
    if (subscriptionGroupConfig.getRetryQueueNums() <= 0) {
        response.setCode(ResponseCode.SUCCESS);
        response.setRemark(null);
        return response;
    }
    String newTopic = MixAll.getRetryTopic(requestHeader.getGroup());
    int queueIdInt = Math.abs(this.random.nextInt() % 99999999) % subscriptionGroupConfig.getRetryQueueNums();
    // 如果是单元化模式，则对 topic 进行设置
    int topicSysFlag = 0;
    if (requestHeader.isUnitMode()) {
        topicSysFlag = TopicSysFlag.buildSysFlag(false, true);
    }
    // 检查topic是否存在
    TopicConfig topicConfig = // 
    this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(// 
    newTopic, // 
    subscriptionGroupConfig.getRetryQueueNums(), PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag);
    if (null == topicConfig) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark("topic[" + newTopic + "] not exist");
        return response;
    }
    // 检查topic权限
    if (!PermName.isWriteable(topicConfig.getPerm())) {
        response.setCode(ResponseCode.NO_PERMISSION);
        response.setRemark(String.format("the topic[%s] sending message is forbidden", newTopic));
        return response;
    }
    // 查询消息，这里如果堆积消息过多，会访问磁盘
    // 另外如果频繁调用，是否会引起gc问题，需要关注 TODO
    MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getOffset());
    if (null == msgExt) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark("look message by offset failed, " + requestHeader.getOffset());
        return response;
    }
    // 构造消息
    final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
    if (null == retryTopic) {
        MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic());
    }
    msgExt.setWaitStoreMsgOK(false);
    // 客户端自动决定定时级别
    int delayLevel = requestHeader.getDelayLevel();
    int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();
    if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {
        maxReconsumeTimes = requestHeader.getMaxReconsumeTimes();
    }
    // 死信消息处理
    if (// 
    msgExt.getReconsumeTimes() >= maxReconsumeTimes || delayLevel < 0) {
        newTopic = MixAll.getDLQTopic(requestHeader.getGroup());
        queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP;
        topicConfig = // 
        this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(// 
        newTopic, // 
        DLQ_NUMS_PER_GROUP, // 死信消息不需要同步，不需要较正。
        PermName.PERM_WRITE, // 死信消息不需要同步，不需要较正。
        0);
        if (null == topicConfig) {
            response.setCode(ResponseCode.SYSTEM_ERROR);
            response.setRemark("topic[" + newTopic + "] not exist");
            return response;
        }
    } else // 继续重试
    {
        if (0 == delayLevel) {
            delayLevel = 3 + msgExt.getReconsumeTimes();
        }
        msgExt.setDelayTimeLevel(delayLevel);
    }
    // 构建 MessageExtBrokerInner
    MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
    msgInner.setTopic(newTopic);
    msgInner.setBody(msgExt.getBody());
    msgInner.setFlag(msgExt.getFlag());
    MessageAccessor.setProperties(msgInner, msgExt.getProperties());
    msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
    msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags()));
    msgInner.setQueueId(queueIdInt);
    msgInner.setSysFlag(msgExt.getSysFlag());
    msgInner.setBornTimestamp(msgExt.getBornTimestamp());
    msgInner.setBornHost(msgExt.getBornHost());
    msgInner.setStoreHost(this.getStoreHost());
    msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1);
    // 保存源生消息的 msgId
    String originMsgId = MessageAccessor.getOriginMessageId(msgExt);
    MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId);
    PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
    if (putMessageResult != null) {
        switch(putMessageResult.getPutMessageStatus()) {
            case PUT_OK:
                // 统计失败重试的Topic
                String backTopic = msgExt.getTopic();
                String correctTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
                if (correctTopic != null) {
                    backTopic = correctTopic;
                }
                this.brokerController.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), backTopic);
                response.setCode(ResponseCode.SUCCESS);
                response.setRemark(null);
                return response;
            default:
                break;
        }
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark(putMessageResult.getPutMessageStatus().name());
        return response;
    }
    response.setCode(ResponseCode.SYSTEM_ERROR);
    response.setRemark("putMessageResult is null");
    return response;
}

Example 15 with MessageExtBrokerInner

use of org.apache.rocketmq.store.MessageExtBrokerInner in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class SendMessageProcessor method sendMessage.

private // 
RemotingCommand sendMessage(// 
final ChannelHandlerContext ctx, // 
final RemotingCommand request, // 
final SendMessageContext sendMessageContext, final SendMessageRequestHeader requestHeader) throws RemotingCommandException {
    // init response
    final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
    final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();
    // 由于有直接返回的逻辑，所以必须要设置
    response.setOpaque(request.getOpaque());
    response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());
    response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));
    log.debug("receive SendMessage request command, {}", request);
    final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
    if (this.brokerController.getMessageStore().now() < startTimstamp) {
        response.setCode(ResponseCode.SYSTEM_ERROR);
        response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp)));
        return response;
    }
    response.setCode(-1);
    super.msgCheck(ctx, requestHeader, response);
    if (response.getCode() != -1) {
        return response;
    }
    final byte[] body = request.getBody();
    int queueIdInt = requestHeader.getQueueId();
    TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
    // 如果队列小于0，从可用队列随机选择
    if (queueIdInt < 0) {
        queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums();
    }
    MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
    msgInner.setTopic(requestHeader.getTopic());
    msgInner.setQueueId(queueIdInt);
    if (!handleRetryAndDLQ(requestHeader, response, request, msgInner, topicConfig)) {
        return response;
    }
    msgInner.setBody(body);
    msgInner.setFlag(requestHeader.getFlag());
    // 设置的属性进行解码  UNIQ_ID就在里面
    MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties()));
    msgInner.setPropertiesString(requestHeader.getProperties());
    msgInner.setBornTimestamp(requestHeader.getBornTimestamp());
    msgInner.setBornHost(ctx.channel().remoteAddress());
    msgInner.setStoreHost(this.getStoreHost());
    msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
    if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
        String traFlag = msgInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
        if (traFlag != null) {
            response.setCode(ResponseCode.NO_PERMISSION);
            response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden");
            return response;
        }
    }
    // 进行存储
    PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
    return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt);
}