Examples with MockId com.github.ambry.store.MockId used on opensource projects

Example 11 with MockId

use of com.github.ambry.store.MockId in project ambry by linkedin.

the class CompositeBlobInfoTest method createKeysAndContentSizes.

/**
 * Creates a list of pairs of StoreKeys and content sizes, where each pair is of a random size
 * @param keySize size of the actual store key
 * @param lowerBound the smallest random content size value allowed
 * @param higherBound the largest random content size value allowed
 * @param numKeys number of keys to populate the list
 * @return a list of pairs of StoreKeys and content sizes, where each pair is of a random size
 */
private List<Pair<StoreKey, Long>> createKeysAndContentSizes(int keySize, int lowerBound, int higherBound, int numKeys) {
    List<Pair<StoreKey, Long>> keysAndContentSizes = new ArrayList<>();
    Random rand = new Random();
    for (int i = 0; i < numKeys; i++) {
        keysAndContentSizes.add(new Pair<>(new MockId(TestUtils.getRandomString(keySize)), (long) rand.nextInt(higherBound - lowerBound) + lowerBound));
    }
    return keysAndContentSizes;
}

Example 12 with MockId

use of com.github.ambry.store.MockId in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatTtlUpdateRecordTest.

/**
 * Tests for {@link TtlUpdateMessageFormatInputStream} in different versions.
 */
@Test
public void messageFormatTtlUpdateRecordTest() throws IOException, MessageFormatException {
    StoreKey key = new MockId("id1");
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long ttlUpdateTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
    long updatedExpiryMs = ttlUpdateTimeMs + TestUtils.RANDOM.nextInt();
    short lifeVersion = 1;
    short oldMessageFormatHeaderVersion = MessageFormatRecord.headerVersionToUse;
    for (short messageFormatHeaderVersion : new short[] { MessageFormatRecord.Message_Header_Version_V2, MessageFormatRecord.Message_Header_Version_V3 }) {
        MessageFormatRecord.headerVersionToUse = messageFormatHeaderVersion;
        MessageFormatInputStream messageFormatStream = new TtlUpdateMessageFormatInputStream(key, accountId, containerId, updatedExpiryMs, ttlUpdateTimeMs, lifeVersion);
        long ttlUpdateRecordSize = MessageFormatRecord.Update_Format_V3.getRecordSize(SubRecord.Type.TTL_UPDATE);
        int headerSize = MessageFormatRecord.getHeaderSizeForVersion(MessageFormatRecord.headerVersionToUse);
        Assert.assertEquals(headerSize + ttlUpdateRecordSize + key.sizeInBytes(), messageFormatStream.getSize());
        checkTtlUpdateMessage(messageFormatStream, ttlUpdateRecordSize, key, accountId, containerId, updatedExpiryMs, ttlUpdateTimeMs, lifeVersion);
    }
    MessageFormatRecord.headerVersionToUse = oldMessageFormatHeaderVersion;
}

Example 13 with MockId

use of com.github.ambry.store.MockId in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatPutNoArgReadTest.

/**
 * Test calling the no-arg read method
 * @throws IOException
 * @throws MessageFormatException
 */
@Test
public void messageFormatPutNoArgReadTest() throws Exception, MessageFormatException {
    StoreKey key = new MockId("id1");
    StoreKeyFactory keyFactory = new MockIdFactory();
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties prop = new BlobProperties(10, "servid", accountId, containerId, false);
    byte[] encryptionKey = new byte[100];
    new Random().nextBytes(encryptionKey);
    byte[] usermetadata = new byte[1000];
    new Random().nextBytes(usermetadata);
    int blobContentSize = 2000;
    byte[] data = new byte[blobContentSize];
    new Random().nextBytes(data);
    ByteBufferInputStream stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
    MessageFormatInputStream messageFormatStream = new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, BlobType.DataBlob);
    TestUtils.validateInputStreamContract(messageFormatStream);
    TestUtils.readInputStreamAndValidateSize(messageFormatStream, messageFormatStream.getSize());
}

Example 14 with MockId

use of com.github.ambry.store.MockId in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatDeleteRecordTest.

/**
 * Tests for {@link DeleteMessageFormatInputStream} in different versions.
 */
@Test
public void messageFormatDeleteRecordTest() throws IOException, MessageFormatException {
    short[] versions = { MessageFormatRecord.Update_Version_V1, MessageFormatRecord.Update_Version_V2, MessageFormatRecord.Update_Version_V3 };
    for (short version : versions) {
        StoreKey key = new MockId("id1");
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        long deletionTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
        short lifeVersion = 1;
        MessageFormatInputStream messageFormatStream;
        short messageHeaderVersionInUse;
        int deleteRecordSize;
        if (version == MessageFormatRecord.Update_Version_V1) {
            messageFormatStream = new DeleteMessageFormatV1InputStream(key, accountId, containerId, deletionTimeMs);
            deleteRecordSize = MessageFormatRecord.Update_Format_V1.getRecordSize();
            messageHeaderVersionInUse = MessageFormatRecord.Message_Header_Version_V1;
            // reset account, container ids and time
            accountId = Account.UNKNOWN_ACCOUNT_ID;
            containerId = Container.UNKNOWN_CONTAINER_ID;
            deletionTimeMs = Utils.Infinite_Time;
        } else if (version == MessageFormatRecord.Update_Version_V2) {
            messageFormatStream = new DeleteMessageFormatV2InputStream(key, accountId, containerId, deletionTimeMs, lifeVersion);
            deleteRecordSize = MessageFormatRecord.Update_Format_V2.getRecordSize();
            messageHeaderVersionInUse = MessageFormatRecord.headerVersionToUse;
        } else {
            messageFormatStream = new DeleteMessageFormatInputStream(key, accountId, containerId, deletionTimeMs, lifeVersion);
            deleteRecordSize = MessageFormatRecord.Update_Format_V3.getRecordSize(SubRecord.Type.DELETE);
            messageHeaderVersionInUse = MessageFormatRecord.headerVersionToUse;
        }
        int headerSize = MessageFormatRecord.getHeaderSizeForVersion(messageHeaderVersionInUse);
        Assert.assertEquals("Unexpected size for version " + version, headerSize + deleteRecordSize + key.sizeInBytes(), messageFormatStream.getSize());
        // check header
        byte[] headerOutput = new byte[headerSize];
        messageFormatStream.read(headerOutput);
        ByteBuffer headerBuf = ByteBuffer.wrap(headerOutput);
        Assert.assertEquals(messageHeaderVersionInUse, headerBuf.getShort());
        if (messageHeaderVersionInUse == MessageFormatRecord.Message_Header_Version_V3) {
            Assert.assertEquals(lifeVersion, headerBuf.getShort());
        }
        Assert.assertEquals(deleteRecordSize, headerBuf.getLong());
        // read encryption key relative offset
        if (messageHeaderVersionInUse >= MessageFormatRecord.Message_Header_Version_V2) {
            Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        }
        // blob properties relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        // delete record relative offset. This is the only relative offset with a valid value.
        Assert.assertEquals(headerSize + key.sizeInBytes(), headerBuf.getInt());
        // user metadata relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        // blob relative offset
        Assert.assertEquals(MessageFormatRecord.Message_Header_Invalid_Relative_Offset, headerBuf.getInt());
        Crc32 crc = new Crc32();
        crc.update(headerOutput, 0, headerSize - MessageFormatRecord.Crc_Size);
        Assert.assertEquals(crc.getValue(), headerBuf.getLong());
        // verify handle
        byte[] handleOutput = new byte[key.sizeInBytes()];
        messageFormatStream.read(handleOutput);
        Assert.assertArrayEquals(handleOutput, key.toBytes());
        // check delete record
        UpdateRecord updateRecord = MessageFormatRecord.deserializeUpdateRecord(messageFormatStream);
        Assert.assertEquals("Type of update record not DELETE", SubRecord.Type.DELETE, updateRecord.getType());
        Assert.assertNotNull("DeleteSubRecord should not be null", updateRecord.getDeleteSubRecord());
        Assert.assertEquals("AccountId mismatch", accountId, updateRecord.getAccountId());
        Assert.assertEquals("ContainerId mismatch", containerId, updateRecord.getContainerId());
        Assert.assertEquals("DeletionTime mismatch", deletionTimeMs, updateRecord.getUpdateTimeInMs());
    }
}

Example 15 with MockId

use of com.github.ambry.store.MockId in project ambry by linkedin.

the class MessageFormatSendTest method doSendWriteSingleMessageTest.

/**
 * Helper method for testing single message sends.
 * @param encryptionKey the encryption key to include in the message while writing it.
 * @param expectedEncryptionKey the key expected when reading the sent message.
 */
private void doSendWriteSingleMessageTest(ByteBuffer encryptionKey, ByteBuffer expectedEncryptionKey) throws Exception {
    String serviceId = "serviceId";
    String ownerId = "owner";
    String contentType = "bin";
    short accountId = 10;
    short containerId = 2;
    byte[] blob = TestUtils.getRandomBytes(10000);
    byte[] userMetadata = TestUtils.getRandomBytes(2000);
    StoreKey storeKey = new MockId("012345678910123456789012");
    BlobProperties properties = new BlobProperties(blob.length, serviceId, ownerId, contentType, false, 100, accountId, containerId, encryptionKey != null, null, null, null);
    MessageFormatInputStream putStream;
    MessageFormatRecord.MessageHeader_Format header;
    if (putFormat.equals(PutMessageFormatInputStream.class.getSimpleName())) {
        header = getHeader(new PutMessageFormatInputStream(storeKey, encryptionKey == null ? null : encryptionKey.duplicate(), properties, ByteBuffer.wrap(userMetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blob.length, BlobType.DataBlob));
        putStream = new PutMessageFormatInputStream(storeKey, encryptionKey, properties, ByteBuffer.wrap(userMetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blob.length, BlobType.DataBlob);
    } else {
        header = getHeader(new PutMessageFormatBlobV1InputStream(storeKey, properties, ByteBuffer.wrap(userMetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blob.length, BlobType.DataBlob));
        putStream = new PutMessageFormatBlobV1InputStream(storeKey, properties, ByteBuffer.wrap(userMetadata), new ByteBufferInputStream(ByteBuffer.wrap(blob)), blob.length, BlobType.DataBlob);
    }
    ByteBuffer buf1 = ByteBuffer.allocate((int) putStream.getSize());
    putStream.read(buf1.array());
    ArrayList<ByteBuffer> listbuf = new ArrayList<ByteBuffer>();
    listbuf.add(buf1);
    ArrayList<StoreKey> storeKeys = new ArrayList<StoreKey>();
    storeKeys.add(storeKey);
    MockMessageReadSet readSet = new MockMessageReadSet(listbuf, storeKeys);
    MetricRegistry registry = new MetricRegistry();
    MessageFormatMetrics metrics = new MessageFormatMetrics(registry);
    // get all
    MessageFormatSend send = new MessageFormatSend(readSet, MessageFormatFlags.All, metrics, new MockIdFactory());
    Assert.assertEquals(send.sizeInBytes(), putStream.getSize());
    Assert.assertEquals(1, send.getMessageMetadataList().size());
    Assert.assertEquals(null, send.getMessageMetadataList().get(0));
    ByteBuffer bufresult = ByteBuffer.allocate((int) putStream.getSize());
    WritableByteChannel channel = Channels.newChannel(new ByteBufferOutputStream(bufresult));
    while (!send.isSendComplete()) {
        send.writeTo(channel);
    }
    Assert.assertArrayEquals(buf1.array(), bufresult.array());
    Assert.assertTrue(readSet.isPrefetchInfoCorrect(0, readSet.sizeInBytes(0)));
    // get blob
    send = new MessageFormatSend(readSet, MessageFormatFlags.Blob, metrics, new MockIdFactory());
    long blobRecordSize = putFormat.equals(PutMessageFormatInputStream.class.getSimpleName()) ? MessageFormatRecord.Blob_Format_V2.getBlobRecordSize(blob.length) : MessageFormatRecord.Blob_Format_V1.getBlobRecordSize(blob.length);
    Assert.assertEquals(send.sizeInBytes(), blobRecordSize);
    bufresult.clear();
    channel = Channels.newChannel(new ByteBufferOutputStream(bufresult));
    while (!send.isSendComplete()) {
        send.writeTo(channel);
    }
    for (int i = 0; i < blob.length; i++) {
        Assert.assertEquals(blob[i], bufresult.array()[i + (int) blobRecordSize - MessageFormatRecord.Crc_Size - blob.length]);
    }
    if (expectedEncryptionKey == null) {
        Assert.assertEquals(null, send.getMessageMetadataList().get(0));
    } else {
        Assert.assertEquals(expectedEncryptionKey, send.getMessageMetadataList().get(0).getEncryptionKey());
    }
    Assert.assertTrue(readSet.isPrefetchInfoCorrect(0, readSet.sizeInBytes(0)));
    // get user metadata
    send = new MessageFormatSend(readSet, MessageFormatFlags.BlobUserMetadata, metrics, new MockIdFactory());
    long userMetadataRecordSize = MessageFormatRecord.UserMetadata_Format_V1.getUserMetadataSize(ByteBuffer.wrap(userMetadata));
    Assert.assertEquals(send.sizeInBytes(), userMetadataRecordSize);
    bufresult.clear();
    channel = Channels.newChannel(new ByteBufferOutputStream(bufresult));
    while (!send.isSendComplete()) {
        send.writeTo(channel);
    }
    bufresult.flip();
    // read off the header.
    for (int i = 0; i < userMetadataRecordSize - MessageFormatRecord.Crc_Size - userMetadata.length; i++) {
        bufresult.get();
    }
    verifyBlobUserMetadata(userMetadata, bufresult);
    if (expectedEncryptionKey == null) {
        Assert.assertEquals(null, send.getMessageMetadataList().get(0));
    } else {
        Assert.assertEquals(expectedEncryptionKey, send.getMessageMetadataList().get(0).getEncryptionKey());
    }
    Assert.assertTrue(readSet.isPrefetchInfoCorrect(header.getUserMetadataRecordRelativeOffset(), header.getUserMetadataRecordSize()));
    // get blob properties
    send = new MessageFormatSend(readSet, MessageFormatFlags.BlobProperties, metrics, new MockIdFactory());
    long blobPropertiesRecordSize = MessageFormatRecord.BlobProperties_Format_V1.getBlobPropertiesRecordSize(properties);
    Assert.assertEquals(send.sizeInBytes(), blobPropertiesRecordSize);
    bufresult.clear();
    channel = Channels.newChannel(new ByteBufferOutputStream(bufresult));
    while (!send.isSendComplete()) {
        send.writeTo(channel);
    }
    bufresult.flip();
    // read off the header.
    for (int i = 0; i < blobPropertiesRecordSize - MessageFormatRecord.Crc_Size - BlobPropertiesSerDe.getBlobPropertiesSerDeSize(properties); i++) {
        bufresult.get();
    }
    verifyBlobProperties(properties, BlobPropertiesSerDe.getBlobPropertiesFromStream(new DataInputStream(new ByteBufferInputStream(bufresult))));
    Assert.assertEquals(null, send.getMessageMetadataList().get(0));
    Assert.assertTrue(readSet.isPrefetchInfoCorrect(header.getBlobPropertiesRecordRelativeOffset(), header.getBlobPropertiesRecordSize()));
    // get blob info
    send = new MessageFormatSend(readSet, MessageFormatFlags.BlobInfo, metrics, new MockIdFactory());
    Assert.assertEquals(send.sizeInBytes(), blobPropertiesRecordSize + userMetadataRecordSize);
    bufresult.clear();
    channel = Channels.newChannel(new ByteBufferOutputStream(bufresult));
    while (!send.isSendComplete()) {
        send.writeTo(channel);
    }
    bufresult.flip();
    for (int i = 0; i < blobPropertiesRecordSize - MessageFormatRecord.Crc_Size - BlobPropertiesSerDe.getBlobPropertiesSerDeSize(properties); i++) {
        bufresult.get();
    }
    verifyBlobProperties(properties, BlobPropertiesSerDe.getBlobPropertiesFromStream(new DataInputStream(new ByteBufferInputStream(bufresult))));
    for (int i = 0; i < userMetadataRecordSize - userMetadata.length; i++) {
        bufresult.get();
    }
    verifyBlobUserMetadata(userMetadata, bufresult);
    if (expectedEncryptionKey == null) {
        Assert.assertEquals(null, send.getMessageMetadataList().get(0));
    } else {
        Assert.assertEquals(expectedEncryptionKey, send.getMessageMetadataList().get(0).getEncryptionKey());
    }
    Assert.assertTrue(readSet.isPrefetchInfoCorrect(header.getBlobPropertiesRecordRelativeOffset(), header.getBlobPropertiesRecordSize() + header.getUserMetadataRecordSize()));
}