Examples with MockIdFactory com.github.ambry.store.MockIdFactory used on opensource projects

Example 6 with MockIdFactory

use of com.github.ambry.store.MockIdFactory in project ambry by linkedin.

the class MessageFormatRecordTest method testBlobRecordWithMetadataContentV3.

@Test
public void testBlobRecordWithMetadataContentV3() throws IOException, MessageFormatException {
    int numKeys = 5;
    List<StoreKey> keys = getKeys(60, numKeys);
    List<Pair<StoreKey, Long>> keysAndContentSizes = new ArrayList<>();
    long total = 0;
    for (int i = 0; i < numKeys; i++) {
        long randNum = ThreadLocalRandom.current().nextLong(1, 10000000);
        total += randNum;
        keysAndContentSizes.add(new Pair<>(keys.get(i), randNum));
    }
    ByteBuffer metadataContent = getSerializedMetadataContentV3(total, keysAndContentSizes);
    int metadataContentSize = MessageFormatRecord.Metadata_Content_Format_V3.getMetadataContentSize(keys.get(0).sizeInBytes(), keys.size());
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(metadataContentSize);
    ByteBuffer blob = ByteBuffer.allocate((int) blobSize);
    BlobData blobData = getBlobRecordV2(metadataContentSize, BlobType.MetadataBlob, metadataContent, blob);
    Assert.assertEquals(metadataContentSize, blobData.getSize());
    byte[] verify = new byte[metadataContentSize];
    blobData.content().readBytes(verify);
    Assert.assertArrayEquals("Metadata content mismatch", metadataContent.array(), verify);
    blobData.release();
    metadataContent.rewind();
    CompositeBlobInfo compositeBlobInfo = deserializeMetadataContentV3(metadataContent, new MockIdFactory());
    Assert.assertEquals("Total size doesn't match", total, compositeBlobInfo.getTotalSize());
    Assert.assertEquals("List of keys dont match", keys, compositeBlobInfo.getKeys());
    List<CompositeBlobInfo.ChunkMetadata> list = compositeBlobInfo.getChunkMetadataList();
    Assert.assertEquals("ChunkMetadata and input list have different sizes", list.size(), keysAndContentSizes.size());
    long sum = 0;
    for (int i = 0; i < list.size(); i++) {
        Assert.assertEquals(keysAndContentSizes.get(i).getFirst(), list.get(i).getStoreKey());
        Assert.assertEquals((long) keysAndContentSizes.get(i).getSecond(), list.get(i).getSize());
        Assert.assertEquals(sum, list.get(i).getOffset());
        sum += list.get(i).getSize();
    }
}

Example 7 with MockIdFactory

use of com.github.ambry.store.MockIdFactory in project ambry by linkedin.

the class MessageFormatRecordTest method testMetadataContentRecordV3.

@Test
public void testMetadataContentRecordV3() throws IOException, MessageFormatException {
    // Test Metadata Blob V3
    int numKeys = 5;
    List<StoreKey> keys = getKeys(60, numKeys);
    List<Pair<StoreKey, Long>> keysAndContentSizes = new ArrayList<>();
    long total = 0;
    for (int i = 0; i < numKeys; i++) {
        long randNum = ThreadLocalRandom.current().nextLong(1, 10000000);
        total += randNum;
        keysAndContentSizes.add(new Pair<>(keys.get(i), randNum));
    }
    ByteBuffer metadataContent = getSerializedMetadataContentV3(total, keysAndContentSizes);
    CompositeBlobInfo compositeBlobInfo = deserializeMetadataContentV3(metadataContent, new MockIdFactory());
    Assert.assertEquals("Total size doesn't match", total, compositeBlobInfo.getTotalSize());
    Assert.assertEquals("List of keys dont match", keys, compositeBlobInfo.getKeys());
    List<CompositeBlobInfo.ChunkMetadata> list = compositeBlobInfo.getChunkMetadataList();
    Assert.assertEquals("ChunkMetadata and input list have different sizes", list.size(), keysAndContentSizes.size());
    long sum = 0;
    for (int i = 0; i < list.size(); i++) {
        Assert.assertEquals(keysAndContentSizes.get(i).getFirst(), list.get(i).getStoreKey());
        Assert.assertEquals((long) keysAndContentSizes.get(i).getSecond(), list.get(i).getSize());
        Assert.assertEquals(sum, list.get(i).getOffset());
        sum += list.get(i).getSize();
    }
}

Example 8 with MockIdFactory

use of com.github.ambry.store.MockIdFactory in project ambry by linkedin.

the class BlobStoreHardDeleteTest method blobStoreHardDeleteTestUtil.

private void blobStoreHardDeleteTestUtil(short[] blobVersions, BlobType[] blobTypes) throws MessageFormatException, IOException {
    MessageStoreHardDelete hardDelete = new BlobStoreHardDelete();
    StoreKeyFactory keyFactory = new MockIdFactory();
    // create log and write to it
    ReadImp readImp = new ReadImp();
    ArrayList<Long> msgOffsets = readImp.initialize(blobVersions, blobTypes);
    Iterator<HardDeleteInfo> iter = hardDelete.getHardDeleteMessages(readImp.getMessageReadSet(), keyFactory, readImp.getRecoveryInfoList());
    List<HardDeleteInfo> hardDeletedList = new ArrayList<HardDeleteInfo>();
    while (iter.hasNext()) {
        hardDeletedList.add(iter.next());
    }
    // msg1
    HardDeleteInfo hardDeleteInfo = hardDeletedList.get(0);
    Assert.assertNotNull(hardDeleteInfo);
    HardDeleteRecoveryMetadata hardDeleteRecoveryMetadata = new HardDeleteRecoveryMetadata(hardDeleteInfo.getRecoveryInfo(), keyFactory);
    Assert.assertEquals(blobTypes[1], hardDeleteRecoveryMetadata.getBlobType());
    Assert.assertEquals(blobVersions[1], hardDeleteRecoveryMetadata.getBlobRecordVersion());
    // msg2
    hardDeleteInfo = hardDeletedList.get(1);
    Assert.assertNotNull(hardDeleteInfo);
    hardDeleteRecoveryMetadata = new HardDeleteRecoveryMetadata(hardDeleteInfo.getRecoveryInfo(), keyFactory);
    Assert.assertEquals(blobTypes[2], hardDeleteRecoveryMetadata.getBlobType());
    Assert.assertEquals(blobVersions[2], hardDeleteRecoveryMetadata.getBlobRecordVersion());
    // msg5
    hardDeleteInfo = hardDeletedList.get(2);
    Assert.assertNotNull(hardDeleteInfo);
    hardDeleteRecoveryMetadata = new HardDeleteRecoveryMetadata(hardDeleteInfo.getRecoveryInfo(), keyFactory);
    Assert.assertEquals(blobTypes[3], hardDeleteRecoveryMetadata.getBlobType());
    Assert.assertEquals(blobVersions[3], hardDeleteRecoveryMetadata.getBlobRecordVersion());
    // msg6 - NULL.
    Assert.assertNull(hardDeletedList.get(3));
}

Example 9 with MockIdFactory

use of com.github.ambry.store.MockIdFactory in project ambry by linkedin.

the class ValidatingKeyConvertingTransformer method verifySievedTransformedMessage.

private void verifySievedTransformedMessage(MessageSievingInputStream sievedStream, StoreKey key, String serviceId, short accountId, short containerId, byte[] encryptionKey, byte[] usermetadata, byte[] data, short blobVersion, BlobType blobType) throws Exception {
    byte[] headerVersion = new byte[Version_Field_Size_In_Bytes];
    sievedStream.read(headerVersion, 0, Version_Field_Size_In_Bytes);
    short version = ByteBuffer.wrap(headerVersion).getShort();
    if (!isValidHeaderVersion(version)) {
        throw new MessageFormatException("Header version not supported " + version, MessageFormatErrorCodes.Data_Corrupt);
    }
    int headerSize = getHeaderSizeForVersion(version);
    byte[] headerArr = new byte[headerSize];
    ByteBuffer headerBuffer = ByteBuffer.wrap(headerArr, 0, headerSize);
    headerBuffer.putShort(version);
    sievedStream.read(headerArr, Version_Field_Size_In_Bytes, headerSize - Version_Field_Size_In_Bytes);
    MessageHeader_Format header = getMessageHeader(version, headerBuffer);
    byte[] keyInStreamBytes = ((new MockIdFactory()).getStoreKey(new DataInputStream(sievedStream))).toBytes();
    Assert.assertArrayEquals(key.toBytes(), keyInStreamBytes);
    Assert.assertTrue(header.isPutRecord());
    ByteBuffer encryptionKeyInStream = header.hasEncryptionKeyRecord() ? deserializeBlobEncryptionKey(sievedStream) : null;
    BlobProperties propsFromStream = deserializeBlobProperties(sievedStream);
    ByteBuffer userMetadataFromStream = deserializeUserMetadata(sievedStream);
    BlobData blobDataFromStream = deserializeBlob(sievedStream);
    Assert.assertEquals(encryptionKey == null, encryptionKeyInStream == null);
    if (encryptionKey != null) {
        Assert.assertArrayEquals(encryptionKey, encryptionKeyInStream.array());
    }
    Assert.assertEquals(serviceId, propsFromStream.getServiceId());
    Assert.assertEquals(accountId, propsFromStream.getAccountId());
    Assert.assertEquals(containerId, propsFromStream.getContainerId());
    Assert.assertEquals(ByteBuffer.wrap(usermetadata), userMetadataFromStream);
    Assert.assertEquals(blobType, blobDataFromStream.getBlobType());
    ByteBuf byteBuf = blobDataFromStream.content();
    try {
        Assert.assertEquals(Unpooled.wrappedBuffer(data), byteBuf);
    } finally {
        byteBuf.release();
    }
}

Example 10 with MockIdFactory

use of com.github.ambry.store.MockIdFactory in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatPutRecordsTest.

private void messageFormatPutRecordsTest(short blobVersion, BlobType blobType, short headerVersion) throws IOException, MessageFormatException {
    StoreKey key = new MockId("id1");
    StoreKeyFactory keyFactory = new MockIdFactory();
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties prop = new BlobProperties(10, "servid", accountId, containerId, false);
    byte[] encryptionKey = new byte[100];
    new Random().nextBytes(encryptionKey);
    byte[] usermetadata = new byte[1000];
    new Random().nextBytes(usermetadata);
    int blobContentSize = 2000;
    byte[] data = new byte[blobContentSize];
    new Random().nextBytes(data);
    short lifeVersion = 1;
    long blobSize = -1;
    MessageFormatRecord.headerVersionToUse = headerVersion;
    if (blobVersion == MessageFormatRecord.Blob_Version_V1) {
        blobSize = MessageFormatRecord.Blob_Format_V1.getBlobRecordSize(blobContentSize);
    } else if (blobVersion == MessageFormatRecord.Blob_Version_V2 && blobType == BlobType.DataBlob) {
        blobSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(blobContentSize);
    } else if (blobVersion == MessageFormatRecord.Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
        ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
        data = byteBufferBlob.array();
        blobContentSize = data.length;
        blobSize = (int) MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(blobContentSize);
    }
    ByteBufferInputStream stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
    MessageFormatInputStream messageFormatStream = (blobVersion == MessageFormatRecord.Blob_Version_V2) ? new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType, lifeVersion) : new PutMessageFormatBlobV1InputStream(key, prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType);
    int headerSize = MessageFormatRecord.getHeaderSizeForVersion(headerVersion);
    int blobEncryptionKeySize = headerVersion != MessageFormatRecord.Message_Header_Version_V1 ? MessageFormatRecord.BlobEncryptionKey_Format_V1.getBlobEncryptionKeyRecordSize(ByteBuffer.wrap(encryptionKey)) : 0;
    int blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(prop);
    int userMetadataSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(ByteBuffer.wrap(usermetadata));
    Assert.assertEquals(messageFormatStream.getSize(), headerSize + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize + key.sizeInBytes());
    // verify header
    byte[] headerOutput = new byte[headerSize];
    messageFormatStream.read(headerOutput);
    ByteBuffer headerBuf = ByteBuffer.wrap(headerOutput);
    Assert.assertEquals(headerVersion, headerBuf.getShort());
    if (headerVersion == MessageFormatRecord.Message_Header_Version_V3) {
        Assert.assertEquals(lifeVersion, headerBuf.getShort());
    }
    Assert.assertEquals(blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + blobSize, headerBuf.getLong());
    switch(headerVersion) {
        case MessageFormatRecord.Message_Header_Version_V1:
            Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
            Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
            Assert.assertEquals(headerSize + key.sizeInBytes() + blobPropertiesRecordSize, headerBuf.getInt());
            Assert.assertEquals(headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize, headerBuf.getInt());
            break;
        default:
            // case MessageFormatRecord.Message_Header_Version_V2 or V3:
            Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
            Assert.assertEquals(headerSize + key.sizeInBytes() + blobEncryptionKeySize, headerBuf.getInt());
            Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
            Assert.assertEquals(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize, headerBuf.getInt());
            Assert.assertEquals(headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize, headerBuf.getInt());
    }
    Crc32 crc = new Crc32();
    crc.update(headerOutput, 0, headerSize - MessageFormatRecord.Crc_Size);
    Assert.assertEquals(crc.getValue(), headerBuf.getLong());
    // verify handle
    byte[] handleOutput = new byte[key.sizeInBytes()];
    ByteBuffer handleOutputBuf = ByteBuffer.wrap(handleOutput);
    messageFormatStream.read(handleOutput);
    byte[] dest = new byte[key.sizeInBytes()];
    handleOutputBuf.get(dest);
    Assert.assertArrayEquals(dest, key.toBytes());
    // verify encryption key
    if (headerVersion != MessageFormatRecord.Message_Header_Version_V1) {
        byte[] blobEncryptionKeyOutput = new byte[blobEncryptionKeySize];
        ByteBuffer blobEncryptionKeyBuf = ByteBuffer.wrap(blobEncryptionKeyOutput);
        messageFormatStream.read(blobEncryptionKeyOutput);
        Assert.assertEquals(blobEncryptionKeyBuf.getShort(), MessageFormatRecord.Blob_Encryption_Key_V1);
        Assert.assertEquals(blobEncryptionKeyBuf.getInt(), 100);
        dest = new byte[100];
        blobEncryptionKeyBuf.get(dest);
        Assert.assertArrayEquals(dest, encryptionKey);
        crc = new Crc32();
        crc.update(blobEncryptionKeyOutput, 0, blobEncryptionKeySize - MessageFormatRecord.Crc_Size);
        Assert.assertEquals(crc.getValue(), blobEncryptionKeyBuf.getLong());
    }
    // verify blob properties
    byte[] blobPropertiesOutput = new byte[blobPropertiesRecordSize];
    ByteBuffer blobPropertiesBuf = ByteBuffer.wrap(blobPropertiesOutput);
    messageFormatStream.read(blobPropertiesOutput);
    Assert.assertEquals(blobPropertiesBuf.getShort(), 1);
    BlobProperties propOutput = BlobPropertiesSerDe.getBlobPropertiesFromStream(new DataInputStream(new ByteBufferInputStream(blobPropertiesBuf)));
    Assert.assertEquals(10, propOutput.getBlobSize());
    Assert.assertEquals("servid", propOutput.getServiceId());
    Assert.assertEquals("AccountId mismatch", accountId, propOutput.getAccountId());
    Assert.assertEquals("ContainerId mismatch", containerId, propOutput.getContainerId());
    crc = new Crc32();
    crc.update(blobPropertiesOutput, 0, blobPropertiesRecordSize - MessageFormatRecord.Crc_Size);
    Assert.assertEquals(crc.getValue(), blobPropertiesBuf.getLong());
    // verify user metadata
    byte[] userMetadataOutput = new byte[userMetadataSize];
    ByteBuffer userMetadataBuf = ByteBuffer.wrap(userMetadataOutput);
    messageFormatStream.read(userMetadataOutput);
    Assert.assertEquals(userMetadataBuf.getShort(), 1);
    Assert.assertEquals(userMetadataBuf.getInt(), 1000);
    dest = new byte[1000];
    userMetadataBuf.get(dest);
    Assert.assertArrayEquals(dest, usermetadata);
    crc = new Crc32();
    crc.update(userMetadataOutput, 0, userMetadataSize - MessageFormatRecord.Crc_Size);
    Assert.assertEquals(crc.getValue(), userMetadataBuf.getLong());
    // verify blob
    CrcInputStream crcstream = new CrcInputStream(messageFormatStream);
    DataInputStream streamData = new DataInputStream(crcstream);
    Assert.assertEquals(streamData.readShort(), blobVersion);
    if (blobVersion == MessageFormatRecord.Blob_Version_V2) {
        Assert.assertEquals(streamData.readShort(), blobType.ordinal());
    }
    Assert.assertEquals(streamData.readLong(), blobContentSize);
    for (int i = 0; i < blobContentSize; i++) {
        Assert.assertEquals((byte) streamData.read(), data[i]);
    }
    long crcVal = crcstream.getValue();
    Assert.assertEquals(crcVal, streamData.readLong());
    // Verify Blob All
    stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
    messageFormatStream = (blobVersion == MessageFormatRecord.Blob_Version_V2) ? new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key, prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, blobType);
    int totalSize;
    switch(headerVersion) {
        case MessageFormatRecord.Message_Header_Version_V1:
            totalSize = headerSize + key.sizeInBytes() + blobPropertiesRecordSize + userMetadataSize + (int) blobSize;
            break;
        default:
            // case MessageFormatRecord.Message_Header_Version_V2:
            totalSize = headerSize + key.sizeInBytes() + blobEncryptionKeySize + blobPropertiesRecordSize + userMetadataSize + (int) blobSize;
    }
    ByteBuffer allBuf = ByteBuffer.allocate(totalSize);
    messageFormatStream.read(allBuf.array());
    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(new ByteBufferInputStream(allBuf), keyFactory);
    Assert.assertEquals(key, blobAll.getStoreKey());
    Assert.assertArrayEquals(usermetadata, blobAll.getBlobInfo().getUserMetadata());
    Assert.assertEquals(blobContentSize, blobAll.getBlobData().getSize());
    Assert.assertEquals(blobType, blobAll.getBlobData().getBlobType());
    if (headerVersion != MessageFormatRecord.Message_Header_Version_V1) {
        Assert.assertEquals(ByteBuffer.wrap(encryptionKey), blobAll.getBlobEncryptionKey());
    } else {
        Assert.assertEquals(null, blobAll.getBlobEncryptionKey());
    }
    ByteBuf byteBuf = blobAll.getBlobData().content();
    try {
        Assert.assertEquals(Unpooled.wrappedBuffer(data), byteBuf);
    } finally {
        byteBuf.release();
    }
}