Examples with MessageExtBatch org.apache.rocketmq.common.message.MessageExtBatch used on opensource projects

Example 1 with MessageExtBatch

use of org.apache.rocketmq.common.message.MessageExtBatch in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class MappedFile method appendMessagesInner.

/**
 * 向MapedBuffer追加消息<br>
 *
 * @param messageExt     要追加的消息
 * @param cb    用来对消息进行序列化，尤其对于依赖MapedFile Offset的属性进行动态序列化
 * @return 是否成功，写入多少数据
 */
public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb) {
    assert messageExt != null;
    assert cb != null;
    // 获取当前文件已经写到什么位置了
    int currentPos = this.wrotePosition.get();
    // 表示有空余空间
    if (currentPos < this.fileSize) {
        // 缓冲区分片
        // slice() 方法根据现有的缓冲区创建一种 子缓冲区 。也就是说，它创建一个新的缓冲区，新缓冲区与原来的缓冲区的一部分共享数据。
        ByteBuffer byteBuffer = writeBuffer != null ? writeBuffer.slice() : this.mappedByteBuffer.slice();
        // 设置position为当前写位置
        byteBuffer.position(currentPos);
        AppendMessageResult result = null;
        if (messageExt instanceof MessageExtBrokerInner) {
            // 交由callback做append
            result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBrokerInner) messageExt);
        } else if (messageExt instanceof MessageExtBatch) {
            // 交由callback做append
            result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBatch) messageExt);
        } else {
            return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
        }
        // 原子操作
        this.wrotePosition.addAndGet(result.getWroteBytes());
        // 最后一条消息存储时间
        this.storeTimestamp = result.getStoreTimestamp();
        return result;
    }
    // 上层应用应该保证不会走到这里
    log.error("MappedFile.appendMessage return null, wrotePosition: {} fileSize: {}", currentPos, this.fileSize);
    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
}

Example 2 with MessageExtBatch

use of org.apache.rocketmq.common.message.MessageExtBatch in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AppendCallbackTest method testAppendMessageBatchSucc.

@Test
public void testAppendMessageBatchSucc() throws Exception {
    List<Message> messages = new ArrayList<>();
    String topic = "test-topic";
    int queue = 0;
    for (int i = 0; i < 10; i++) {
        Message msg = new Message();
        msg.setBody("body".getBytes());
        msg.setTopic(topic);
        msg.setTags("abc");
        messages.add(msg);
    }
    MessageExtBatch messageExtBatch = new MessageExtBatch();
    messageExtBatch.setTopic(topic);
    messageExtBatch.setQueueId(queue);
    messageExtBatch.setBornTimestamp(System.currentTimeMillis());
    messageExtBatch.setBornHost(new InetSocketAddress("127.0.0.1", 123));
    messageExtBatch.setStoreHost(new InetSocketAddress("127.0.0.1", 124));
    messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
    messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
    ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
    AppendMessageResult allresult = callback.doAppend(0, buff, 1024 * 10, messageExtBatch);
    assertEquals(AppendMessageStatus.PUT_OK, allresult.getStatus());
    assertEquals(0, allresult.getWroteOffset());
    assertEquals(0, allresult.getLogicsOffset());
    assertEquals(buff.position(), allresult.getWroteBytes());
    assertEquals(messages.size(), allresult.getMsgNum());
    Set<String> msgIds = new HashSet<>();
    for (String msgId : allresult.getMsgId().split(",")) {
        assertEquals(32, msgId.length());
        msgIds.add(msgId);
    }
    assertEquals(messages.size(), msgIds.size());
    List<MessageExt> decodeMsgs = MessageDecoder.decodes((ByteBuffer) buff.flip());
    assertEquals(decodeMsgs.size(), decodeMsgs.size());
    long queueOffset = decodeMsgs.get(0).getQueueOffset();
    long storeTimeStamp = decodeMsgs.get(0).getStoreTimestamp();
    for (int i = 0; i < messages.size(); i++) {
        assertEquals(messages.get(i).getTopic(), decodeMsgs.get(i).getTopic());
        assertEquals(new String(messages.get(i).getBody()), new String(decodeMsgs.get(i).getBody()));
        assertEquals(messages.get(i).getTags(), decodeMsgs.get(i).getTags());
        assertEquals(messageExtBatch.getBornHostNameString(), decodeMsgs.get(i).getBornHostNameString());
        assertEquals(messageExtBatch.getBornTimestamp(), decodeMsgs.get(i).getBornTimestamp());
        assertEquals(storeTimeStamp, decodeMsgs.get(i).getStoreTimestamp());
        assertEquals(queueOffset++, decodeMsgs.get(i).getQueueOffset());
    }
}

Example 3 with MessageExtBatch

use of org.apache.rocketmq.common.message.MessageExtBatch in project rocketmq-rocketmq-all-4.1.0-incubating by lirenzuo.

the class AppendCallbackTest method testAppendMessageBatchEndOfFile.

@Test
public void testAppendMessageBatchEndOfFile() throws Exception {
    List<Message> messages = new ArrayList<>();
    String topic = "test-topic";
    int queue = 0;
    for (int i = 0; i < 10; i++) {
        Message msg = new Message();
        msg.setBody("body".getBytes());
        msg.setTopic(topic);
        msg.setTags("abc");
        messages.add(msg);
    }
    MessageExtBatch messageExtBatch = new MessageExtBatch();
    messageExtBatch.setTopic(topic);
    messageExtBatch.setQueueId(queue);
    messageExtBatch.setBornTimestamp(System.currentTimeMillis());
    messageExtBatch.setBornHost(new InetSocketAddress("127.0.0.1", 123));
    messageExtBatch.setStoreHost(new InetSocketAddress("127.0.0.1", 124));
    messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
    messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
    ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
    // encounter end of file when append half of the data
    AppendMessageResult result = callback.doAppend(0, buff, 1000, messageExtBatch);
    assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus());
    assertEquals(0, result.getWroteOffset());
    assertEquals(0, result.getLogicsOffset());
    assertEquals(1000, result.getWroteBytes());
    // write blank size and magic value
    assertEquals(8, buff.position());
    // should have already constructed some message ids
    assertTrue(result.getMsgId().length() > 0);
}

Example 4 with MessageExtBatch

use of org.apache.rocketmq.common.message.MessageExtBatch in project rocketmq by apache.

the class AppendCallbackTest method testAppendMessageBatchEndOfFile.

@Test
public void testAppendMessageBatchEndOfFile() throws Exception {
    List<Message> messages = new ArrayList<>();
    String topic = "test-topic";
    int queue = 0;
    for (int i = 0; i < 10; i++) {
        Message msg = new Message();
        msg.setBody("body".getBytes());
        msg.setTopic(topic);
        msg.setTags("abc");
        messages.add(msg);
    }
    MessageExtBatch messageExtBatch = new MessageExtBatch();
    messageExtBatch.setTopic(topic);
    messageExtBatch.setQueueId(queue);
    messageExtBatch.setBornTimestamp(System.currentTimeMillis());
    messageExtBatch.setBornHost(new InetSocketAddress("127.0.0.1", 123));
    messageExtBatch.setStoreHost(new InetSocketAddress("127.0.0.1", 124));
    messageExtBatch.setBody(MessageDecoder.encodeMessages(messages));
    messageExtBatch.setEncodedBuff(batchEncoder.encode(messageExtBatch));
    ByteBuffer buff = ByteBuffer.allocate(1024 * 10);
    // encounter end of file when append half of the data
    AppendMessageResult result = callback.doAppend(0, buff, 1000, messageExtBatch);
    assertEquals(AppendMessageStatus.END_OF_FILE, result.getStatus());
    assertEquals(0, result.getWroteOffset());
    assertEquals(0, result.getLogicsOffset());
    assertEquals(1000, result.getWroteBytes());
    // write blank size and magic value
    assertEquals(8, buff.position());
    // should have already constructed some message ids
    assertTrue(result.getMsgId().length() > 0);
}

Example 5 with MessageExtBatch

use of org.apache.rocketmq.common.message.MessageExtBatch in project rocketmq by apache.

the class MappedFile method appendMessagesInner.

public AppendMessageResult appendMessagesInner(final MessageExt messageExt, final AppendMessageCallback cb) {
    assert messageExt != null;
    assert cb != null;
    int currentPos = this.wrotePosition.get();
    if (currentPos < this.fileSize) {
        ByteBuffer byteBuffer = writeBuffer != null ? writeBuffer.slice() : this.mappedByteBuffer.slice();
        byteBuffer.position(currentPos);
        AppendMessageResult result = null;
        if (messageExt instanceof MessageExtBrokerInner) {
            result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBrokerInner) messageExt);
        } else if (messageExt instanceof MessageExtBatch) {
            result = cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, (MessageExtBatch) messageExt);
        } else {
            return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
        }
        this.wrotePosition.addAndGet(result.getWroteBytes());
        this.storeTimestamp = result.getStoreTimestamp();
        return result;
    }
    log.error("MappedFile.appendMessage return null, wrotePosition: {} fileSize: {}", currentPos, this.fileSize);
    return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
}