Examples with Crc32 com.github.ambry.utils.Crc32 used on opensource projects

Example 6 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class MessageFormatTestUtils method getBlobContentForMetadataBlob.

/**
 * Creates a test data for metadata blob content, followed by creating the actual blob record V2
 * @param blobSize size of the metadata content
 * @return entire blob content as a {@link ByteBuffer}
 */
public static ByteBuffer getBlobContentForMetadataBlob(int blobSize) {
    ByteBuffer blobContent = ByteBuffer.allocate(blobSize);
    new Random().nextBytes(blobContent.array());
    int size = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(blobSize);
    ByteBuffer entireBlob = ByteBuffer.allocate(size);
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(entireBlob, blobSize, BlobType.MetadataBlob);
    entireBlob.put(blobContent);
    Crc32 crc = new Crc32();
    crc.update(entireBlob.array(), 0, entireBlob.position());
    entireBlob.putLong(crc.getValue());
    entireBlob.flip();
    return entireBlob;
}

Example 7 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class MessageFormatRecordTest method serializeBlobPropertiesV3Record.

/**
 * Serialize {@link BlobProperties} in version {@link BlobPropertiesSerDe#VERSION_3}
 * @param outputBuffer {@link ByteBuffer} to serialize the {@link BlobProperties}
 * @param properties {@link BlobProperties} to be serialized
 */
private void serializeBlobPropertiesV3Record(ByteBuffer outputBuffer, BlobProperties properties) {
    int startOffset = outputBuffer.position();
    outputBuffer.putShort(BlobProperties_Version_V1);
    putBlobPropertiesToBufferV3(outputBuffer, properties);
    Crc32 crc = new Crc32();
    crc.update(outputBuffer.array(), startOffset, getBlobPropertiesV3RecordSize(properties) - Crc_Size);
    outputBuffer.putLong(crc.getValue());
}

Example 8 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class MessageFormatRecordTest method serializeBlobPropertiesV2Record.

/**
 * Serialize {@link BlobProperties} in version {@link BlobPropertiesSerDe#VERSION_2}
 * @param outputBuffer {@link ByteBuffer} to serialize the {@link BlobProperties}
 * @param properties {@link BlobProperties} to be serialized
 */
private void serializeBlobPropertiesV2Record(ByteBuffer outputBuffer, BlobProperties properties) {
    int startOffset = outputBuffer.position();
    outputBuffer.putShort(BlobProperties_Version_V1);
    putBlobPropertiesToBufferV2(outputBuffer, properties);
    Crc32 crc = new Crc32();
    crc.update(outputBuffer.array(), startOffset, getBlobPropertiesV2RecordSize(properties) - Crc_Size);
    outputBuffer.putLong(crc.getValue());
}

Example 9 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class PutMessageFormatInputStream method createStreamWithMessageHeaderV1.

/**
 * Helper method to create a stream without encryption key record. This is the default currently, but once all nodes
 * once all nodes in a cluster understand reading messages with encryption key record, and writing in the new format
 * is enabled, this method can be removed.
 */
private void createStreamWithMessageHeaderV1(StoreKey key, BlobProperties blobProperties, ByteBuffer userMetadata, InputStream blobStream, long streamSize, BlobType blobType) throws MessageFormatException {
    int headerSize = MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize();
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(streamSize);
    buffer = ByteBuffer.allocate(headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize + (int) (blobSize - streamSize - MessageFormatRecord.Crc_Size));
    MessageFormatRecord.MessageHeader_Format_V1.serializeHeader(buffer, blobPropertiesRecordSize + userMetadataSize + blobSize, headerSize + key.sizeInBytes(), MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerSize + key.sizeInBytes() + blobPropertiesRecordSize, headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize);
    buffer.put(key.toBytes());
    MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(buffer, blobProperties);
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(buffer, userMetadata);
    int bufferBlobStart = buffer.position();
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(buffer, streamSize, blobType);
    Crc32 crc = new Crc32();
    crc.update(buffer.array(), bufferBlobStart, buffer.position() - bufferBlobStart);
    stream = new CrcInputStream(crc, blobStream);
    streamLength = streamSize;
    messageLength = buffer.capacity() + streamLength + MessageFormatRecord.Crc_Size;
    buffer.flip();
}

Example 10 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class PutMessageFormatInputStream method createStreamWithMessageHeader.

/**
 * Helper method to create a stream with encryption key record. This will be the standard once all nodes in a cluster
 * understand reading messages with encryption key record.
 */
private void createStreamWithMessageHeader(StoreKey key, ByteBuffer blobEncryptionKey, BlobProperties blobProperties, ByteBuffer userMetadata, InputStream blobStream, long streamSize, BlobType blobType, short lifeVersion) throws MessageFormatException {
    int headerSize = MessageFormatRecord.getHeaderSizeForVersion(MessageFormatRecord.headerVersionToUse);
    int blobEncryptionKeySize = blobEncryptionKey == null ? 0 : MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(blobEncryptionKey);
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(streamSize);
    buffer = ByteBuffer.allocate(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + (int) (blobSize - streamSize - MessageFormatRecord.Crc_Size));
    long totalSize = blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize;
    int blobEncryptionKeyRecordRelativeOffset = blobEncryptionKey == null ? MessageFormatRecord.Message_Header_Invalid_Relative_Offset : headerSize + key.sizeInBytes();
    int blobPropertiesRecordRelativeOffset = blobEncryptionKey == null ? headerSize + key.sizeInBytes() : blobEncryptionKeyRecordRelativeOffset + blobEncryptionKeySize;
    int updateRecordRelativeOffset = MessageFormatRecord.Message_Header_Invalid_Relative_Offset;
    int userMetadataRecordRelativeOffset = blobPropertiesRecordRelativeOffset + blobPropertiesRecordSize;
    int blobRecordRelativeOffset = userMetadataRecordRelativeOffset + userMetadataSize;
    if (MessageFormatRecord.headerVersionToUse == MessageFormatRecord.Message_Header_Version_V2) {
        MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(buffer, totalSize, blobEncryptionKeyRecordRelativeOffset, blobPropertiesRecordRelativeOffset, updateRecordRelativeOffset, userMetadataRecordRelativeOffset, blobRecordRelativeOffset);
    } else {
        MessageFormatRecord.MessageHeader_Format_V3.serializeHeader(buffer, lifeVersion, totalSize, blobEncryptionKeyRecordRelativeOffset, blobPropertiesRecordRelativeOffset, updateRecordRelativeOffset, userMetadataRecordRelativeOffset, blobRecordRelativeOffset);
    }
    buffer.put(key.toBytes());
    if (blobEncryptionKey != null) {
        MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(buffer, blobEncryptionKey);
    }
    MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(buffer, blobProperties);
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(buffer, userMetadata);
    int bufferBlobStart = buffer.position();
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(buffer, streamSize, blobType);
    Crc32 crc = new Crc32();
    crc.update(buffer.array(), bufferBlobStart, buffer.position() - bufferBlobStart);
    stream = new CrcInputStream(crc, blobStream);
    streamLength = streamSize;
    messageLength = buffer.capacity() + streamLength + MessageFormatRecord.Crc_Size;
    buffer.flip();
}