Examples with Crc32 com.github.ambry.utils.Crc32 used on opensource projects

Example 1 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class PutMessageFormatInputStream method createStreamWithMessageHeaderV2.

/**
 * Helper method to create a stream with encryption key record. This will be the standard once all nodes in a cluster
 * understand reading messages with encryption key record.
 */
private void createStreamWithMessageHeaderV2(StoreKey key, ByteBuffer blobEncryptionKey, BlobProperties blobProperties, ByteBuffer userMetadata, InputStream blobStream, long streamSize, BlobType blobType) throws MessageFormatException {
    int headerSize = MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize();
    int blobEncryptionKeySize = blobEncryptionKey == null ? 0 : MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(blobEncryptionKey);
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(blobProperties);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(userMetadata);
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(streamSize);
    buffer = ByteBuffer.allocate(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + (int) (blobSize - streamSize - MessageFormatRecord.Crc_Size));
    long totalSize = blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize;
    int blobEncryptionKeyRecordRelativeOffset = blobEncryptionKey == null ? MessageFormatRecord.Message_Header_Invalid_Relative_Offset : headerSize + key.sizeInBytes();
    int blobPropertiesRecordRelativeOffset = blobEncryptionKey == null ? headerSize + key.sizeInBytes() : blobEncryptionKeyRecordRelativeOffset + blobEncryptionKeySize;
    int deleteRecordRelativeOffset = MessageFormatRecord.Message_Header_Invalid_Relative_Offset;
    int userMetadataRecordRelativeOffset = blobPropertiesRecordRelativeOffset + blobPropertiesRecordSize;
    int blobRecordRelativeOffset = userMetadataRecordRelativeOffset + userMetadataSize;
    MessageFormatRecord.MessageHeader_Format_V2.serializeHeader(buffer, totalSize, blobEncryptionKeyRecordRelativeOffset, blobPropertiesRecordRelativeOffset, deleteRecordRelativeOffset, userMetadataRecordRelativeOffset, blobRecordRelativeOffset);
    buffer.put(key.toBytes());
    if (blobEncryptionKey != null) {
        MessageFormatRecord.BlobEncryptionKey_Format_V1.serializeBlobEncryptionKeyRecord(buffer, blobEncryptionKey);
    }
    MessageFormatRecord.BlobProperties_Format_V1.serializeBlobPropertiesRecord(buffer, blobProperties);
    MessageFormatRecord.UserMetadata_Format_V1.serializeUserMetadataRecord(buffer, userMetadata);
    int bufferBlobStart = buffer.position();
    MessageFormatRecord.Blob_Format_V2.serializePartialBlobRecord(buffer, streamSize, blobType);
    Crc32 crc = new Crc32();
    crc.update(buffer.array(), bufferBlobStart, buffer.position() - bufferBlobStart);
    stream = new CrcInputStream(crc, blobStream);
    streamLength = streamSize;
    messageLength = buffer.capacity() + streamLength + MessageFormatRecord.Crc_Size;
    buffer.flip();
}

Example 2 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class RestUtilsTest method getUserMetadataFromByteArrayComplexTest.

/**
 * Tests deserializing user metadata from byte array
 */
@Test
public void getUserMetadataFromByteArrayComplexTest() {
    Map<String, String> userMetadataMap = null;
    // user metadata of size 1 byte
    byte[] userMetadataByteArray = new byte[1];
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // user metadata with just the version
    userMetadataByteArray = new byte[4];
    ByteBuffer byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // user metadata with wrong version
    userMetadataByteArray = new byte[4];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 3);
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // 0 sized user metadata
    userMetadataByteArray = new byte[12];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(4);
    byteBuffer.putInt(0);
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // wrong size
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    String key = "key1";
    byte[] keyInBytes = key.getBytes(RestUtils.CHARSET);
    int keyLength = keyInBytes.length;
    byteBuffer.putInt(21);
    byteBuffer.putInt(1);
    byteBuffer.putInt(keyLength);
    byteBuffer.put(keyInBytes);
    String value = "value1";
    byte[] valueInBytes = value.getBytes(RestUtils.CHARSET);
    int valueLength = valueInBytes.length;
    byteBuffer.putInt(valueLength);
    byteBuffer.put(valueInBytes);
    Crc32 crc32 = new Crc32();
    crc32.update(userMetadataByteArray, 0, userMetadataByteArray.length - 8);
    byteBuffer.putLong(crc32.getValue());
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // wrong total number of entries
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(22);
    byteBuffer.putInt(2);
    byteBuffer.putInt(keyLength);
    byteBuffer.put(keyInBytes);
    byteBuffer.putInt(valueLength);
    byteBuffer.put(valueInBytes);
    crc32 = new Crc32();
    crc32.update(userMetadataByteArray, 0, userMetadataByteArray.length - 8);
    byteBuffer.putLong(crc32.getValue());
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // diff key length
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(22);
    byteBuffer.putInt(1);
    byteBuffer.putInt(keyLength + 1);
    byteBuffer.put(keyInBytes);
    byteBuffer.putInt(valueLength);
    byteBuffer.put(valueInBytes);
    crc32 = new Crc32();
    crc32.update(userMetadataByteArray, 0, userMetadataByteArray.length - 8);
    byteBuffer.putLong(crc32.getValue());
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // diff value length
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(22);
    byteBuffer.putInt(1);
    byteBuffer.putInt(keyLength);
    byteBuffer.put(keyInBytes);
    byteBuffer.putInt(valueLength + 1);
    byteBuffer.put(valueInBytes);
    crc32 = new Crc32();
    crc32.update(userMetadataByteArray, 0, userMetadataByteArray.length - 8);
    byteBuffer.putLong(crc32.getValue());
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // no crc
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(22);
    byteBuffer.putInt(1);
    byteBuffer.putInt(keyLength);
    byteBuffer.put(keyInBytes);
    byteBuffer.putInt(valueLength);
    byteBuffer.put(valueInBytes);
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // wrong crc
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(22);
    byteBuffer.putInt(1);
    byteBuffer.putInt(keyLength);
    byteBuffer.put(keyInBytes);
    byteBuffer.putInt(valueLength);
    byteBuffer.put(valueInBytes);
    crc32 = new Crc32();
    crc32.update(userMetadataByteArray, 0, userMetadataByteArray.length - 8);
    byteBuffer.putLong(crc32.getValue() - 1);
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertNull("UserMetadata should have been null ", userMetadataMap);
    // correct crc
    userMetadataByteArray = new byte[36];
    byteBuffer = ByteBuffer.wrap(userMetadataByteArray);
    byteBuffer.putShort((short) 1);
    byteBuffer.putInt(22);
    byteBuffer.putInt(1);
    byteBuffer.putInt(keyLength);
    byteBuffer.put(keyInBytes);
    byteBuffer.putInt(valueLength);
    byteBuffer.put(valueInBytes);
    crc32 = new Crc32();
    crc32.update(userMetadataByteArray, 0, userMetadataByteArray.length - 8);
    byteBuffer.putLong(crc32.getValue());
    userMetadataMap = RestUtils.buildUserMetadata(userMetadataByteArray);
    assertEquals("Sizes don't match ", userMetadataMap.size(), 1);
    assertTrue("User metadata " + RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + key + " not found in user metadata ", userMetadataMap.containsKey(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + key));
    assertEquals("User metadata " + RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + key + " value don't match ", value, userMetadataMap.get(RestUtils.Headers.USER_META_DATA_HEADER_PREFIX + key));
}

Example 3 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatDeleteRecordTest.

/**
 * Tests for {@link DeleteMessageFormatInputStream} in different versions.
 */
@Test
public void messageFormatDeleteRecordTest() throws IOException, MessageFormatException {
    short[] versions = { MessageFormatRecord.Update_Version_V1, MessageFormatRecord.Update_Version_V2, MessageFormatRecord.Update_Version_V3 };
    for (short version : versions) {
        StoreKey key = new MockId("id1");
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        long deletionTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
        short lifeVersion = 1;
        MessageFormatInputStream messageFormatStream;
        short messageHeaderVersionInUse;
        int deleteRecordSize;
        if (version == MessageFormatRecord.Update_Version_V1) {
            messageFormatStream = new DeleteMessageFormatV1InputStream(key, accountId, containerId, deletionTimeMs);
            deleteRecordSize = MessageFormatRecord.Update_Format_V1.getRecordSize();
            messageHeaderVersionInUse = MessageFormatRecord.Message_Header_Version_V1;
            // reset account, container ids and time
            accountId = Account.UNKNOWN_ACCOUNT_ID;
            containerId = Container.UNKNOWN_CONTAINER_ID;
            deletionTimeMs = Utils.Infinite_Time;
        } else if (version == MessageFormatRecord.Update_Version_V2) {
            messageFormatStream = new DeleteMessageFormatV2InputStream(key, accountId, containerId, deletionTimeMs, lifeVersion);
            deleteRecordSize = MessageFormatRecord.Update_Format_V2.getRecordSize();
            messageHeaderVersionInUse = MessageFormatRecord.headerVersionToUse;
        } else {
            messageFormatStream = new DeleteMessageFormatInputStream(key, accountId, containerId, deletionTimeMs, lifeVersion);
            deleteRecordSize = MessageFormatRecord.Update_Format_V3.getRecordSize(SubRecord.Type.DELETE);
            messageHeaderVersionInUse = MessageFormatRecord.headerVersionToUse;
        }
        int headerSize = MessageFormatRecord.getHeaderSizeForVersion(messageHeaderVersionInUse);
        Assert.assertEquals("Unexpected size for version " + version, headerSize + deleteRecordSize + key.sizeInBytes(), messageFormatStream.getSize());
        // check header
        byte[] headerOutput = new byte[headerSize];
        messageFormatStream.read(headerOutput);
        ByteBuffer headerBuf = ByteBuffer.wrap(headerOutput);
        Assert.assertEquals(messageHeaderVersionInUse, headerBuf.getShort());
        if (messageHeaderVersionInUse == MessageFormatRecord.Message_Header_Version_V3) {
            Assert.assertEquals(lifeVersion, headerBuf.getShort());
        }
        Assert.assertEquals(deleteRecordSize, headerBuf.getLong());
        // read encryption key relative offset
        if (messageHeaderVersionInUse >= MessageFormatRecord.Message_Header_Version_V2) {
            Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        }
        // blob properties relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        // delete record relative offset. This is the only relative offset with a valid value.
        Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
        // user metadata relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        // blob relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        Crc32 crc = new Crc32();
        crc.update(headerOutput, 0, headerSize - MessageFormatRecord.Crc_Size);
        Assert.assertEquals(crc.getValue(), headerBuf.getLong());
        // verify handle
        byte[] handleOutput = new byte[key.sizeInBytes()];
        messageFormatStream.read(handleOutput);
        Assert.assertArrayEquals(handleOutput, key.toBytes());
        // check delete record
        UpdateRecord updateRecord = MessageFormatRecord.deserializeUpdateRecord(messageFormatStream);
        Assert.assertEquals("Type of update record not DELETE", SubRecord.Type.DELETE, updateRecord.getType());
        Assert.assertNotNull("DeleteSubRecord should not be null", updateRecord.getDeleteSubRecord());
        Assert.assertEquals("AccountId mismatch", accountId, updateRecord.getAccountId());
        Assert.assertEquals("ContainerId mismatch", containerId, updateRecord.getContainerId());
        Assert.assertEquals("DeletionTime mismatch", deletionTimeMs, updateRecord.getUpdateTimeInMs());
    }
}

Example 4 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class MessageFormatSendTest method sendWriteTestWithBadId.

@Test
public void sendWriteTestWithBadId() throws IOException, MessageFormatException {
    // add header,system metadata, user metadata and data to the buffers
    ByteBuffer buf1 = ByteBuffer.allocate(1010);
    // fill header
    // version
    buf1.putShort((short) 1);
    // total size
    buf1.putLong(950);
    // put relative offsets
    // blob property relative offset
    buf1.putInt(60);
    // delete relative offset
    buf1.putInt(-1);
    // user metadata relative offset
    buf1.putInt(81);
    // data relative offset
    buf1.putInt(191);
    Crc32 crc = new Crc32();
    crc.update(buf1.array(), 0, buf1.position());
    // crc
    buf1.putLong(crc.getValue());
    // blob id
    String id = new String("012345678910123456789012");
    buf1.putShort((short) id.length());
    buf1.put(id.getBytes());
    // blob property version
    buf1.putShort((short) 1);
    String attribute1 = "ttl";
    String attribute2 = "del";
    // ttl name
    buf1.put(attribute1.getBytes());
    // ttl value
    buf1.putLong(12345);
    // delete name
    buf1.put(attribute2.getBytes());
    byte b = 1;
    // delete flag
    buf1.put(b);
    // crc
    buf1.putInt(456);
    // user metadata version
    buf1.putShort((short) 1);
    buf1.putInt(100);
    byte[] usermetadata = new byte[100];
    new Random().nextBytes(usermetadata);
    buf1.put(usermetadata);
    buf1.putInt(123);
    // blob version
    buf1.putShort((short) 0);
    // blob size
    buf1.putLong(805);
    // blob
    byte[] data = new byte[805];
    new Random().nextBytes(data);
    buf1.put(data);
    // blob crc
    buf1.putInt(123);
    buf1.flip();
    ArrayList<ByteBuffer> listbuf = new ArrayList<ByteBuffer>();
    listbuf.add(buf1);
    ArrayList<StoreKey> storeKeys = new ArrayList<StoreKey>();
    storeKeys.add(new MockId("012345678910123223233456789012"));
    MessageReadSet readSet = new MockMessageReadSet(listbuf, storeKeys);
    MetricRegistry registry = new MetricRegistry();
    MessageFormatMetrics metrics = new MessageFormatMetrics(registry);
    // get all
    MessageFormatSend send = new MessageFormatSend(readSet, MessageFormatFlags.All, metrics, new MockIdFactory());
    Assert.assertEquals(send.sizeInBytes(), 1010);
    ByteBuffer bufresult = ByteBuffer.allocate(1010);
    WritableByteChannel channel1 = Channels.newChannel(new ByteBufferOutputStream(bufresult));
    while (!send.isSendComplete()) {
        send.writeTo(channel1);
    }
    Assert.assertArrayEquals(buf1.array(), bufresult.array());
    try {
        // get blob
        MessageFormatSend send1 = new MessageFormatSend(readSet, MessageFormatFlags.Blob, metrics, new MockIdFactory());
        Assert.fail("Exception is expected");
    } catch (MessageFormatException e) {
        Assert.assertTrue(e.getErrorCode() == MessageFormatErrorCodes.Store_Key_Id_MisMatch);
    }
}

Example 5 with Crc32

use of com.github.ambry.utils.Crc32 in project ambry by linkedin.

the class MessageFormatRecordTest method serializeBlobPropertiesV1Record.

/**
 * Serialize {@link BlobProperties} in version {@link BlobPropertiesSerDe#VERSION_1}
 * @param outputBuffer {@link ByteBuffer} to serialize the {@link BlobProperties}
 * @param properties {@link BlobProperties} to be serialized
 */
private void serializeBlobPropertiesV1Record(ByteBuffer outputBuffer, BlobProperties properties) {
    int startOffset = outputBuffer.position();
    outputBuffer.putShort(BlobProperties_Version_V1);
    putBlobPropertiesToBufferV1(outputBuffer, properties);
    Crc32 crc = new Crc32();
    crc.update(outputBuffer.array(), startOffset, getBlobPropertiesV1RecordSize(properties) - Crc_Size);
    outputBuffer.putLong(crc.getValue());
}