Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 1 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method serializeMessageInfoAndMetadataList.

/**
 * Serialize this object into the given buffer.
 * @param outputBuffer the ByteBuffer to serialize into.
 */
void serializeMessageInfoAndMetadataList(ByteBuffer outputBuffer) {
    outputBuffer.putInt(messageInfoList == null ? 0 : messageInfoList.size());
    if (messageInfoList != null) {
        ListIterator<MessageInfo> infoListIterator = messageInfoList.listIterator();
        ListIterator<MessageMetadata> metadataListIterator = messageMetadataList.listIterator();
        while (infoListIterator.hasNext()) {
            MessageInfo messageInfo = infoListIterator.next();
            MessageMetadata messageMetadata = metadataListIterator.next();
            outputBuffer.put(messageInfo.getStoreKey().toBytes());
            outputBuffer.putLong(messageInfo.getSize());
            outputBuffer.putLong(messageInfo.getExpirationTimeInMs());
            outputBuffer.put(messageInfo.isDeleted() ? DELETED : (byte) ~DELETED);
            if (version < VERSION_1 || version > VERSION_4) {
                throw new IllegalArgumentException("Unknown version in MessageInfoList " + version);
            }
            if (version > VERSION_1) {
                Long crc = messageInfo.getCrc();
                if (crc != null) {
                    outputBuffer.put(FIELD_PRESENT);
                    outputBuffer.putLong(crc);
                } else {
                    outputBuffer.put((byte) ~FIELD_PRESENT);
                }
            }
            if (version > VERSION_2) {
                outputBuffer.putShort(messageInfo.getAccountId());
                outputBuffer.putShort(messageInfo.getContainerId());
                outputBuffer.putLong(messageInfo.getOperationTimeMs());
            }
            if (version > VERSION_3) {
                if (messageMetadata != null) {
                    outputBuffer.put(FIELD_PRESENT);
                    messageMetadata.serializeMessageMetadata(outputBuffer);
                } else {
                    outputBuffer.put((byte) ~FIELD_PRESENT);
                }
            }
        }
    }
}

Example 2 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTest method getPutMessage.

/**
 * Constructs an entire message with header, blob properties, user metadata and blob content.
 * @param id id for which the message has to be constructed.
 * @param accountId accountId of the blob
 * @param containerId containerId of the blob
 * @param enableEncryption {@code true} if encryption needs to be enabled. {@code false} otherwise
 * @return a {@link Pair} of {@link ByteBuffer} and {@link MessageInfo} representing the entire message and the
 *         associated {@link MessageInfo}
 * @throws MessageFormatException
 * @throws IOException
 */
private Pair<ByteBuffer, MessageInfo> getPutMessage(StoreKey id, short accountId, short containerId, boolean enableEncryption) throws MessageFormatException, IOException {
    int blobSize = TestUtils.RANDOM.nextInt(500) + 501;
    int userMetadataSize = TestUtils.RANDOM.nextInt(blobSize / 2);
    int encryptionKeySize = TestUtils.RANDOM.nextInt(blobSize / 4);
    byte[] blob = new byte[blobSize];
    byte[] usermetadata = new byte[userMetadataSize];
    byte[] encryptionKey = enableEncryption ? new byte[encryptionKeySize] : null;
    TestUtils.RANDOM.nextBytes(blob);
    TestUtils.RANDOM.nextBytes(usermetadata);
    BlobProperties blobProperties = new BlobProperties(blobSize, "test", accountId, containerId, encryptionKey != null);
    MessageFormatInputStream stream = new PutMessageFormatInputStream(id, encryptionKey == null ? null : ByteBuffer.wrap(encryptionKey), blobProperties, ByteBuffer.wrap(usermetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blobSize);
    byte[] message = Utils.readBytesFromStream(stream, (int) stream.getSize());
    return new Pair<>(ByteBuffer.wrap(message), new MessageInfo(id, message.length, Utils.Infinite_Time, accountId, containerId, blobProperties.getCreationTimeInMs()));
}

Example 3 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTest method addPutMessagesToReplicasOfPartition.

/**
 * For the given partitionId, constructs put messages and adds them to the given lists.
 * @param partitionId the {@link PartitionId} to use for generating the {@link StoreKey} of the message.
 * @param hosts the list of {@link Host} all of which will be populated with the messages.
 * @param count the number of messages to construct and add.
 * @return the list of blobs ids that were generated.
 * @throws MessageFormatException
 * @throws IOException
 */
private List<StoreKey> addPutMessagesToReplicasOfPartition(PartitionId partitionId, List<Host> hosts, int count) throws MessageFormatException, IOException {
    List<StoreKey> ids = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
        boolean toEncrypt = i % 2 == 0;
        BlobId id = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, ClusterMapUtils.UNKNOWN_DATACENTER_ID, accountId, containerId, partitionId, toEncrypt);
        ids.add(id);
        Pair<ByteBuffer, MessageInfo> putMsgInfo = getPutMessage(id, accountId, containerId, toEncrypt);
        for (Host host : hosts) {
            host.addMessage(partitionId, putMsgInfo.getSecond(), putMsgInfo.getFirst().duplicate());
        }
    }
    return ids;
}

Example 4 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicationTest method addDeleteMessagesToReplicasOfPartition.

/**
 * For the given partitionId, constructs delete messages and adds them to the given lists.
 * @param partitionId the {@link PartitionId} to use for generating the {@link StoreKey} of the message.
 * @param id the {@link StoreKey} to create a delete message for.
 * @param hosts the list of {@link Host} all of which will be populated with the messages.
 * @throws MessageFormatException
 * @throws IOException
 */
private void addDeleteMessagesToReplicasOfPartition(PartitionId partitionId, StoreKey id, List<Host> hosts) throws MessageFormatException, IOException {
    MessageInfo putMsg = getMessageInfo(id, hosts.get(0).infosByPartition.get(partitionId), false);
    long deletionTimeMs = System.currentTimeMillis();
    ByteBuffer buffer = getDeleteMessage(id, putMsg.getAccountId(), putMsg.getContainerId(), deletionTimeMs);
    for (Host host : hosts) {
        host.addMessage(partitionId, new MessageInfo(id, buffer.remaining(), true, putMsg.getAccountId(), putMsg.getContainerId(), deletionTimeMs), buffer.duplicate());
    }
}

Example 5 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class HardDeleteRecoveryMetadata method getMessageInfo.

@Override
public MessageInfo getMessageInfo(Read read, long offset, StoreKeyFactory storeKeyFactory) throws IOException {
    try {
        // read message header
        ByteBuffer headerVersion = ByteBuffer.allocate(Version_Field_Size_In_Bytes);
        read.readInto(headerVersion, offset);
        offset += headerVersion.capacity();
        headerVersion.flip();
        short version = headerVersion.getShort();
        MessageHeader_Format headerFormat;
        ReadInputStream stream;
        long endOffset;
        if (!isValidHeaderVersion(version)) {
            throw new MessageFormatException("Version not known while reading message - " + version, MessageFormatErrorCodes.Unknown_Format_Version);
        }
        ByteBuffer header = ByteBuffer.allocate(getHeaderSizeForVersion(version));
        header.putShort(version);
        read.readInto(header, offset);
        offset += header.capacity() - headerVersion.capacity();
        header.flip();
        headerFormat = getMessageHeader(version, header);
        headerFormat.verifyHeader();
        endOffset = offset + headerFormat.getPayloadRelativeOffset() + headerFormat.getMessageSize();
        stream = new ReadInputStream(read, offset, endOffset);
        StoreKey key = storeKeyFactory.getStoreKey(new DataInputStream(stream));
        if (headerFormat.hasEncryptionKeyRecord()) {
            deserializeBlobEncryptionKey(stream);
        }
        // read the appropriate type of message based on the relative offset that is set
        if (headerFormat.isPutRecord()) {
            BlobProperties properties = deserializeBlobProperties(stream);
            return new MessageInfo(key, header.capacity() + key.sizeInBytes() + headerFormat.getMessageSize(), Utils.addSecondsToEpochTime(properties.getCreationTimeInMs(), properties.getTimeToLiveInSeconds()), properties.getAccountId(), properties.getContainerId(), properties.getCreationTimeInMs());
        } else {
            DeleteRecord deleteRecord = deserializeDeleteRecord(stream);
            return new MessageInfo(key, header.capacity() + key.sizeInBytes() + headerFormat.getMessageSize(), true, deleteRecord.getAccountId(), deleteRecord.getContainerId(), deleteRecord.getDeletionTimeInMs());
        }
    } catch (MessageFormatException e) {
        // log in case where we were not able to parse a message.
        throw new IOException("Message format exception while parsing messages ", e);
    } catch (IndexOutOfBoundsException e) {
        // log in case where were not able to read a complete message.
        throw new IOException("Trying to read more than the available bytes");
    }
}