Examples with StoreKey com.github.ambry.store.StoreKey used on opensource projects

Example 21 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatRecordTest method testBlobRecordWithMetadataContentV2.

@Test
public void testBlobRecordWithMetadataContentV2() throws IOException, MessageFormatException {
    // Test Blob V2 with actual metadata blob V2
    // construct metadata blob
    List<StoreKey> keys = getKeys(60, 5);
    int[] chunkSizes = { ThreadLocalRandom.current().nextInt(1, Integer.MAX_VALUE), 15 };
    long[] totalSizes = { (long) keys.size() * chunkSizes[0], ((long) keys.size() * chunkSizes[1]) - 11 };
    for (int i = 0; i < chunkSizes.length; i++) {
        ByteBuffer metadataContent = getSerializedMetadataContentV2(chunkSizes[i], totalSizes[i], keys);
        int metadataContentSize = MessageFormatRecord.Metadata_Content_Format_V2.getMetadataContentSize(keys.get(0).sizeInBytes(), keys.size());
        long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(metadataContentSize);
        ByteBuffer blob = ByteBuffer.allocate((int) blobSize);
        BlobData blobData = getBlobRecordV2(metadataContentSize, BlobType.MetadataBlob, metadataContent, blob);
        Assert.assertEquals(metadataContentSize, blobData.getSize());
        byte[] verify = new byte[metadataContentSize];
        blobData.content().readBytes(verify);
        Assert.assertArrayEquals("Metadata content mismatch", metadataContent.array(), verify);
        blobData.release();
        // deserialize and check for metadata contents
        metadataContent.rewind();
        CompositeBlobInfo compositeBlobInfo = deserializeMetadataContentV2(metadataContent, new MockIdFactory());
        Assert.assertEquals("Chunk size doesn't match", chunkSizes[i], compositeBlobInfo.getChunkSize());
        Assert.assertEquals("Total size doesn't match", totalSizes[i], compositeBlobInfo.getTotalSize());
        Assert.assertEquals("List of keys dont match", keys, compositeBlobInfo.getKeys());
        testBlobCorruption(blob, blobSize, metadataContentSize);
    }
}

Example 22 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatRecordTest method testBlobRecordWithMetadataContentV3.

@Test
public void testBlobRecordWithMetadataContentV3() throws IOException, MessageFormatException {
    int numKeys = 5;
    List<StoreKey> keys = getKeys(60, numKeys);
    List<Pair<StoreKey, Long>> keysAndContentSizes = new ArrayList<>();
    long total = 0;
    for (int i = 0; i < numKeys; i++) {
        long randNum = ThreadLocalRandom.current().nextLong(1, 10000000);
        total += randNum;
        keysAndContentSizes.add(new Pair<>(keys.get(i), randNum));
    }
    ByteBuffer metadataContent = getSerializedMetadataContentV3(total, keysAndContentSizes);
    int metadataContentSize = MessageFormatRecord.Metadata_Content_Format_V3.getMetadataContentSize(keys.get(0).sizeInBytes(), keys.size());
    long blobSize = MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(metadataContentSize);
    ByteBuffer blob = ByteBuffer.allocate((int) blobSize);
    BlobData blobData = getBlobRecordV2(metadataContentSize, BlobType.MetadataBlob, metadataContent, blob);
    Assert.assertEquals(metadataContentSize, blobData.getSize());
    byte[] verify = new byte[metadataContentSize];
    blobData.content().readBytes(verify);
    Assert.assertArrayEquals("Metadata content mismatch", metadataContent.array(), verify);
    blobData.release();
    metadataContent.rewind();
    CompositeBlobInfo compositeBlobInfo = deserializeMetadataContentV3(metadataContent, new MockIdFactory());
    Assert.assertEquals("Total size doesn't match", total, compositeBlobInfo.getTotalSize());
    Assert.assertEquals("List of keys dont match", keys, compositeBlobInfo.getKeys());
    List<CompositeBlobInfo.ChunkMetadata> list = compositeBlobInfo.getChunkMetadataList();
    Assert.assertEquals("ChunkMetadata and input list have different sizes", list.size(), keysAndContentSizes.size());
    long sum = 0;
    for (int i = 0; i < list.size(); i++) {
        Assert.assertEquals(keysAndContentSizes.get(i).getFirst(), list.get(i).getStoreKey());
        Assert.assertEquals((long) keysAndContentSizes.get(i).getSecond(), list.get(i).getSize());
        Assert.assertEquals(sum, list.get(i).getOffset());
        sum += list.get(i).getSize();
    }
}

Example 23 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageFormatRecordTest method testMetadataContentRecordV3.

@Test
public void testMetadataContentRecordV3() throws IOException, MessageFormatException {
    // Test Metadata Blob V3
    int numKeys = 5;
    List<StoreKey> keys = getKeys(60, numKeys);
    List<Pair<StoreKey, Long>> keysAndContentSizes = new ArrayList<>();
    long total = 0;
    for (int i = 0; i < numKeys; i++) {
        long randNum = ThreadLocalRandom.current().nextLong(1, 10000000);
        total += randNum;
        keysAndContentSizes.add(new Pair<>(keys.get(i), randNum));
    }
    ByteBuffer metadataContent = getSerializedMetadataContentV3(total, keysAndContentSizes);
    CompositeBlobInfo compositeBlobInfo = deserializeMetadataContentV3(metadataContent, new MockIdFactory());
    Assert.assertEquals("Total size doesn't match", total, compositeBlobInfo.getTotalSize());
    Assert.assertEquals("List of keys dont match", keys, compositeBlobInfo.getKeys());
    List<CompositeBlobInfo.ChunkMetadata> list = compositeBlobInfo.getChunkMetadataList();
    Assert.assertEquals("ChunkMetadata and input list have different sizes", list.size(), keysAndContentSizes.size());
    long sum = 0;
    for (int i = 0; i < list.size(); i++) {
        Assert.assertEquals(keysAndContentSizes.get(i).getFirst(), list.get(i).getStoreKey());
        Assert.assertEquals((long) keysAndContentSizes.get(i).getSecond(), list.get(i).getSize());
        Assert.assertEquals(sum, list.get(i).getOffset());
        sum += list.get(i).getSize();
    }
}

Example 24 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class DeserializedBlob method deserializeBlobAll.

/**
 * Deserialize a complete blob record into a {@link BlobAll} object.
 * @param stream the {@link InputStream} from which to read the blob record.
 * @param storeKeyFactory the factory for parsing store keys.
 * @return a {@link BlobAll} object with the {@link BlobInfo} and {@link BlobData} for the blob.
 * @throws IOException
 * @throws MessageFormatException
 */
public static BlobAll deserializeBlobAll(InputStream stream, StoreKeyFactory storeKeyFactory) throws IOException, MessageFormatException {
    DataInputStream inputStream = stream instanceof DataInputStream ? (DataInputStream) stream : new DataInputStream(stream);
    short headerVersion = inputStream.readShort();
    ByteBuffer headerBuf;
    MessageHeader_Format header;
    switch(headerVersion) {
        case Message_Header_Version_V1:
            headerBuf = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize());
            headerBuf.putShort(headerVersion);
            inputStream.read(headerBuf.array(), Version_Field_Size_In_Bytes, MessageHeader_Format_V1.getHeaderSize() - Version_Field_Size_In_Bytes);
            headerBuf.rewind();
            header = new MessageHeader_Format_V1(headerBuf);
            break;
        case Message_Header_Version_V2:
            headerBuf = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V2.getHeaderSize());
            headerBuf.putShort(headerVersion);
            inputStream.read(headerBuf.array(), Version_Field_Size_In_Bytes, MessageHeader_Format_V2.getHeaderSize() - Version_Field_Size_In_Bytes);
            headerBuf.rewind();
            header = new MessageHeader_Format_V2(headerBuf);
            break;
        case Message_Header_Version_V3:
            headerBuf = ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V3.getHeaderSize());
            headerBuf.putShort(headerVersion);
            inputStream.read(headerBuf.array(), Version_Field_Size_In_Bytes, MessageHeader_Format_V3.getHeaderSize() - Version_Field_Size_In_Bytes);
            headerBuf.rewind();
            header = new MessageHeader_Format_V3(headerBuf);
            break;
        default:
            throw new MessageFormatException("Message header version not supported", MessageFormatErrorCodes.Unknown_Format_Version);
    }
    header.verifyHeader();
    StoreKey storeKey = storeKeyFactory.getStoreKey(inputStream);
    ByteBuffer blobEncryptionKey = null;
    if (header.hasEncryptionKeyRecord()) {
        blobEncryptionKey = deserializeBlobEncryptionKey(stream);
    }
    BlobProperties blobProperties = deserializeBlobProperties(stream);
    byte[] userMetadata = deserializeUserMetadata(stream).array();
    BlobData blobData = deserializeBlob(stream);
    return new BlobAll(storeKey, blobEncryptionKey, new BlobInfo(blobProperties, userMetadata), blobData);
}

Example 25 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class CompositeBlobInfoTest method testV2CompositeBlobInfoCtor.

/**
 * Tests various kinds of inputs for the V2 {@link CompositeBlobInfo}
 */
@Test
public void testV2CompositeBlobInfoCtor() {
    List<StoreKey> keys = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        keys.add(new MockId(TestUtils.getRandomString(60)));
    }
    // creates 1 10 byte blob, so not enough keys
    invalidV2CompositeBlobInfoCtor(100, 10, keys);
    // creates 9 100 byte blobs, so too many keys
    invalidV2CompositeBlobInfoCtor(100, 900, keys);
    // creates 9 100 byte blobs and 1 1 byte blob
    validV2CompositeBlobInfoCtor(100, 901, keys);
    // creates 10 100 byte blobs
    validV2CompositeBlobInfoCtor(100, 1000, keys);
    // creates 10 100 byte blobs and 1 1 byte blob, so not enough keys
    invalidV2CompositeBlobInfoCtor(100, 1001, keys);
    // rest are a mix of negative and 0 values for chunkSize and totalSize, all invalid
    invalidV2CompositeBlobInfoCtor(0, 0, keys);
    invalidV2CompositeBlobInfoCtor(1, 0, keys);
    invalidV2CompositeBlobInfoCtor(0, 1, keys);
    invalidV2CompositeBlobInfoCtor(-1, 1, keys);
    invalidV2CompositeBlobInfoCtor(-1, -1, keys);
    invalidV2CompositeBlobInfoCtor(1, -1, keys);
}