Examples with MessageMetadata com.github.ambry.messageformat.MessageMetadata used on opensource projects

Example 1 with MessageMetadata

use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method serializeMessageInfoAndMetadataList.

/**
 * Serialize this object into the given buffer.
 * @param outputBuffer the ByteBuffer to serialize into.
 */
void serializeMessageInfoAndMetadataList(ByteBuffer outputBuffer) {
    outputBuffer.putInt(messageInfoList == null ? 0 : messageInfoList.size());
    if (messageInfoList != null) {
        ListIterator<MessageInfo> infoListIterator = messageInfoList.listIterator();
        ListIterator<MessageMetadata> metadataListIterator = messageMetadataList.listIterator();
        while (infoListIterator.hasNext()) {
            MessageInfo messageInfo = infoListIterator.next();
            MessageMetadata messageMetadata = metadataListIterator.next();
            outputBuffer.put(messageInfo.getStoreKey().toBytes());
            outputBuffer.putLong(messageInfo.getSize());
            outputBuffer.putLong(messageInfo.getExpirationTimeInMs());
            outputBuffer.put(messageInfo.isDeleted() ? DELETED : (byte) ~DELETED);
            if (version < VERSION_1 || version > VERSION_4) {
                throw new IllegalArgumentException("Unknown version in MessageInfoList " + version);
            }
            if (version > VERSION_1) {
                Long crc = messageInfo.getCrc();
                if (crc != null) {
                    outputBuffer.put(FIELD_PRESENT);
                    outputBuffer.putLong(crc);
                } else {
                    outputBuffer.put((byte) ~FIELD_PRESENT);
                }
            }
            if (version > VERSION_2) {
                outputBuffer.putShort(messageInfo.getAccountId());
                outputBuffer.putShort(messageInfo.getContainerId());
                outputBuffer.putLong(messageInfo.getOperationTimeMs());
            }
            if (version > VERSION_3) {
                if (messageMetadata != null) {
                    outputBuffer.put(FIELD_PRESENT);
                    messageMetadata.serializeMessageMetadata(outputBuffer);
                } else {
                    outputBuffer.put((byte) ~FIELD_PRESENT);
                }
            }
        }
    }
}

Example 2 with MessageMetadata

use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method getMessageInfoAndMetadataListSize.

/**
 * @return the size in bytes that serialization of this MessageInfoAndMetadataList will result in.
 */
int getMessageInfoAndMetadataListSize() {
    int size = 0;
    if (version >= VERSION_5) {
        // version
        size += Short.BYTES;
    }
    // num elements in list
    size += Integer.BYTES;
    if (messageInfoList == null) {
        return size;
    }
    ListIterator<MessageInfo> infoListIterator = messageInfoList.listIterator();
    ListIterator<MessageMetadata> metadataListIterator = messageMetadataList.listIterator();
    while (infoListIterator.hasNext()) {
        MessageInfo messageInfo = infoListIterator.next();
        MessageMetadata messageMetadata = metadataListIterator.next();
        size += messageInfo.getStoreKey().sizeInBytes();
        // message size
        size += Long.BYTES;
        // expiration time
        size += Long.BYTES;
        // whether deleted
        size += Byte.BYTES;
        if (version < VERSION_1 || version > VERSION_MAX) {
            throw new IllegalArgumentException("Unknown version in MessageInfoList " + version);
        }
        if (version >= VERSION_5) {
            // whether ttl updated
            size += Byte.BYTES;
        }
        if (version >= VERSION_6) {
            // whether undelete
            size += Byte.BYTES;
        }
        if (version > VERSION_1) {
            // whether crc is present
            size += Byte.BYTES;
            if (messageInfo.getCrc() != null) {
                // crc
                size += Long.BYTES;
            }
        }
        if (version > VERSION_2) {
            // accountId
            size += Short.BYTES;
            // containerId
            size += Short.BYTES;
            // operationTime
            size += Long.BYTES;
        }
        if (version >= VERSION_6) {
            // lifeVersion
            size += Short.BYTES;
        }
        if (version > VERSION_3) {
            // whether message metadata is present.
            size += Byte.BYTES;
            if (messageMetadata != null) {
                size += messageMetadata.sizeInBytes();
            }
        }
    }
    return size;
}

Example 3 with MessageMetadata

use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method serializeMessageInfoAndMetadataList.

/**
 * Serialize this object into the given buffer.
 * @param outputBuf the {@link ByteBuf} to serialize into.
 */
void serializeMessageInfoAndMetadataList(ByteBuf outputBuf) {
    if (version >= VERSION_5) {
        outputBuf.writeShort(version);
    }
    outputBuf.writeInt(messageInfoList == null ? 0 : messageInfoList.size());
    if (messageInfoList != null) {
        ListIterator<MessageInfo> infoListIterator = messageInfoList.listIterator();
        ListIterator<MessageMetadata> metadataListIterator = messageMetadataList.listIterator();
        while (infoListIterator.hasNext()) {
            MessageInfo messageInfo = infoListIterator.next();
            MessageMetadata messageMetadata = metadataListIterator.next();
            outputBuf.writeBytes(messageInfo.getStoreKey().toBytes());
            outputBuf.writeLong(messageInfo.getSize());
            outputBuf.writeLong(messageInfo.getExpirationTimeInMs());
            outputBuf.writeByte(messageInfo.isDeleted() ? UPDATED : (byte) ~UPDATED);
            if (version < VERSION_1 || version > VERSION_MAX) {
                throw new IllegalArgumentException("Unknown version in MessageInfoList " + version);
            }
            if (version >= VERSION_5) {
                outputBuf.writeByte(messageInfo.isTtlUpdated() ? UPDATED : (byte) ~UPDATED);
            }
            if (version >= VERSION_6) {
                outputBuf.writeByte(messageInfo.isUndeleted() ? UPDATED : (byte) ~UPDATED);
            }
            if (version > VERSION_1) {
                Long crc = messageInfo.getCrc();
                if (crc != null) {
                    outputBuf.writeByte(FIELD_PRESENT);
                    outputBuf.writeLong(crc);
                } else {
                    outputBuf.writeByte((byte) ~FIELD_PRESENT);
                }
            }
            if (version > VERSION_2) {
                outputBuf.writeShort(messageInfo.getAccountId());
                outputBuf.writeShort(messageInfo.getContainerId());
                outputBuf.writeLong(messageInfo.getOperationTimeMs());
            }
            if (version >= VERSION_6) {
                outputBuf.writeShort(messageInfo.getLifeVersion());
            }
            if (version > VERSION_3) {
                if (messageMetadata != null) {
                    outputBuf.writeByte(FIELD_PRESENT);
                    messageMetadata.serializeMessageMetadata(outputBuf);
                } else {
                    outputBuf.writeByte((byte) ~FIELD_PRESENT);
                }
            }
        }
    }
}

Example 4 with MessageMetadata

use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.

the class MessageInfoAndMetadataListSerde method deserializeMessageInfoAndMetadataList.

/**
 * Deserialize the given stream and return the MessageInfo and Metadata lists.
 * @param stream the stream to deserialize from.
 * @param map the clustermap to use.
 * @param versionToDeserializeIn the SerDe version to use to deserialize.
 * @return the deserialized {@link MessageInfoAndMetadataListSerde}.
 * @throws IOException if an I/O error occurs while reading from the stream.
 */
static MessageInfoAndMetadataListSerde deserializeMessageInfoAndMetadataList(DataInputStream stream, ClusterMap map, short versionToDeserializeIn) throws IOException {
    if (versionToDeserializeIn >= VERSION_5) {
        short versionFromStream = stream.readShort();
        if (versionFromStream != versionToDeserializeIn) {
            throw new IllegalArgumentException("Argument provided [" + versionToDeserializeIn + "] and stream [" + versionFromStream + "] disagree on version");
        }
    } else {
        versionToDeserializeIn = versionToDeserializeIn == DETERMINE_VERSION ? stream.readShort() : versionToDeserializeIn;
    }
    int messageCount = stream.readInt();
    ArrayList<MessageInfo> messageInfoList = new ArrayList<>(messageCount);
    ArrayList<MessageMetadata> messageMetadataList = new ArrayList<>(messageCount);
    for (int i = 0; i < messageCount; i++) {
        BlobId id = new BlobId(stream, map);
        long size = stream.readLong();
        long ttl = stream.readLong();
        boolean isDeleted = stream.readByte() == UPDATED;
        boolean isTtlUpdated = false;
        boolean isUndeleted = false;
        short lifeVersion = 0;
        Long crc = null;
        short accountId = Account.UNKNOWN_ACCOUNT_ID;
        short containerId = Container.UNKNOWN_CONTAINER_ID;
        long operationTime = Utils.Infinite_Time;
        if (versionToDeserializeIn < VERSION_1 || versionToDeserializeIn > VERSION_MAX) {
            throw new IllegalArgumentException("Unknown version to deserialize MessageInfoList " + versionToDeserializeIn);
        }
        if (versionToDeserializeIn >= VERSION_5) {
            isTtlUpdated = stream.readByte() == UPDATED;
        }
        if (versionToDeserializeIn >= VERSION_6) {
            isUndeleted = stream.readByte() == UPDATED;
        }
        if (versionToDeserializeIn > VERSION_1) {
            crc = stream.readByte() == FIELD_PRESENT ? stream.readLong() : null;
        }
        if (versionToDeserializeIn > VERSION_2) {
            accountId = stream.readShort();
            containerId = stream.readShort();
            operationTime = stream.readLong();
        }
        if (versionToDeserializeIn >= VERSION_6) {
            lifeVersion = stream.readShort();
        }
        messageInfoList.add(new MessageInfo(id, size, isDeleted, isTtlUpdated, isUndeleted, ttl, crc, accountId, containerId, operationTime, lifeVersion));
        if (versionToDeserializeIn > VERSION_3) {
            MessageMetadata messageMetadata = stream.readByte() == FIELD_PRESENT ? MessageMetadata.deserializeMessageMetadata(stream) : null;
            messageMetadataList.add(messageMetadata);
        } else {
            messageMetadataList.add(null);
        }
    }
    return new MessageInfoAndMetadataListSerde(messageInfoList, messageMetadataList, versionToDeserializeIn);
}

Example 5 with MessageMetadata

use of com.github.ambry.messageformat.MessageMetadata in project ambry by linkedin.

the class MessageInfoAndMetadataListSerDeTest method testSerDe.

@Test
public void testSerDe() throws Exception {
    MockClusterMap mockMap = new MockClusterMap();
    MockPartitionId partitionId = new MockPartitionId();
    short[] accountIds = { 100, 101, 102, 103 };
    short[] containerIds = { 10, 11, 12, 13 };
    boolean[] isDeletedVals = { false, true, false, true };
    boolean[] isTtlUpdatedVals = { true, false, false, true };
    boolean[] isUndeletedVals = { true, false, false, true };
    short[] lifeVersionVals = { 1, 2, 3, 4 };
    Long[] crcs = { null, 100L, Long.MIN_VALUE, Long.MAX_VALUE };
    StoreKey[] keys = { new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[0], containerIds[0], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[1], containerIds[1], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[2], containerIds[2], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[3], containerIds[3], partitionId, false, BlobId.BlobDataType.DATACHUNK) };
    long[] blobSizes = { 1024, 2048, 4096, 8192 };
    long[] times = { SystemTime.getInstance().milliseconds(), SystemTime.getInstance().milliseconds() - 1, SystemTime.getInstance().milliseconds() + TimeUnit.SECONDS.toMillis(5), Utils.Infinite_Time };
    MessageMetadata[] messageMetadata = new MessageMetadata[4];
    messageMetadata[0] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(100)));
    messageMetadata[1] = new MessageMetadata(null);
    messageMetadata[2] = null;
    messageMetadata[3] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(200)));
    List<MessageInfo> messageInfoList = new ArrayList<>(4);
    List<MessageMetadata> messageMetadataList = new ArrayList<>(4);
    for (int i = 0; i < 4; i++) {
        messageInfoList.add(new MessageInfo(keys[i], blobSizes[i], isDeletedVals[i], isTtlUpdatedVals[i], isUndeletedVals[i], times[i], crcs[i], accountIds[i], containerIds[i], times[i], lifeVersionVals[i]));
        messageMetadataList.add(messageMetadata[i]);
    }
    // Serialize and then deserialize
    MessageInfoAndMetadataListSerde messageInfoAndMetadataListSerde = new MessageInfoAndMetadataListSerde(messageInfoList, messageMetadataList, serDeVersion);
    ByteBuf buffer = Unpooled.buffer(messageInfoAndMetadataListSerde.getMessageInfoAndMetadataListSize());
    messageInfoAndMetadataListSerde.serializeMessageInfoAndMetadataList(buffer);
    if (serDeVersion >= MessageInfoAndMetadataListSerde.VERSION_5) {
        // should fail if the wrong version is provided
        try {
            MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, (short) (serDeVersion - 1));
            Assert.fail("Should have failed to deserialize");
        } catch (IllegalArgumentException e) {
        // expected. Nothing to do
        }
    }
    buffer.readerIndex(0);
    MessageInfoAndMetadataListSerde messageInfoAndMetadataList = MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, serDeVersion);
    // Verify
    List<MessageInfo> responseMessageInfoList = messageInfoAndMetadataList.getMessageInfoList();
    List<MessageMetadata> responseMessageMetadataList = messageInfoAndMetadataList.getMessageMetadataList();
    Assert.assertEquals(4, responseMessageInfoList.size());
    Assert.assertEquals(4, responseMessageMetadataList.size());
    for (int i = 0; i < 4; i++) {
        assertMessageInfoEquality(messageInfoList.get(i), responseMessageInfoList.get(i));
        assertMessageMetadataEquality(messageMetadataList.get(i), responseMessageMetadataList.get(i));
    }
}