Examples with BlobProperties com.github.ambry.messageformat.BlobProperties used on opensource projects

Example 26 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class VcrBackupTest method basicTest.

/**
 * Basic test to make sure VCR can backup with HelixVcrCluster.
 */
@Test
public void basicTest() throws Exception {
    List<BlobId> blobIds = sendBlobToDataNode(dataNode, 10);
    // Start the VCR and CloudBackupManager
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, clusterMapPort, 12410, 12510, serverSSLProps, vcrHelixStateModelFactoryClass, true);
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // Waiting for backup done
    assertTrue("Did not backup all blobs in 2 minutes", latchBasedInMemoryCloudDestination.awaitUpload(2, TimeUnit.MINUTES));
    // Verify a blob by making a http2 request.
    MockClusterMap clusterMap = mockCluster.getClusterMap();
    SSLConfig clientSSLConfig = new SSLConfig(new VerifiableProperties(clientSSLProps));
    ConnectedChannel channel = ServerTestUtil.getBlockingChannelBasedOnPortType(new Port(clusterMap.getDataNodes().get(0).getHttp2Port(), PortType.HTTP2), "localhost", null, clientSSLConfig);
    BlobId blobToVerify = blobIds.get(0);
    ArrayList<BlobId> idList = new ArrayList<>(Arrays.asList(blobToVerify));
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobToVerify.getPartition(), idList);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
    DataInputStream stream = channel.sendAndReceive(getRequest1).getInputStream();
    GetResponse resp1 = GetResponse.readFrom(stream, clusterMap);
    try {
        BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
        // Do a simple check
        assertEquals(blobSize, propertyOutput.getBlobSize());
        releaseNettyBufUnderneathStream(stream);
    } catch (MessageFormatException e) {
        fail();
    }
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
}

Example 27 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class Verifier method run.

@Override
public void run() {
    try {
        List<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        while (requestsVerified.get() != totalRequests.get() && !cancelTest.get()) {
            Payload payload = payloadQueue.poll(1000, TimeUnit.MILLISECONDS);
            if (payload != null) {
                notificationSystem.awaitBlobCreations(payload.blobId);
                for (MockDataNodeId dataNodeId : clusterMap.getDataNodes()) {
                    ConnectedChannel channel1 = null;
                    try {
                        BlobId blobId = new BlobId(payload.blobId, clusterMap);
                        Port port = new Port(portType == PortType.PLAINTEXT ? dataNodeId.getPort() : dataNodeId.getSSLPort(), portType);
                        channel1 = connectionPool.checkOutConnection("localhost", port, 10000);
                        ArrayList<BlobId> ids = new ArrayList<BlobId>();
                        ids.add(blobId);
                        partitionRequestInfoList.clear();
                        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
                        partitionRequestInfoList.add(partitionRequestInfo);
                        GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        DataInputStream stream = channel1.receive().getInputStream();
                        GetResponse resp = GetResponse.readFrom(stream, clusterMap);
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println(dataNodeId.getHostname() + " " + dataNodeId.getPort() + " " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                                if (propertyOutput.getBlobSize() != payload.blobProperties.getBlobSize()) {
                                    String exceptionMsg = "blob size not matching " + " expected " + payload.blobProperties.getBlobSize() + " actual " + propertyOutput.getBlobSize();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (!propertyOutput.getServiceId().equals(payload.blobProperties.getServiceId())) {
                                    String exceptionMsg = "service id not matching " + " expected " + payload.blobProperties.getServiceId() + " actual " + propertyOutput.getBlobSize();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (propertyOutput.getAccountId() != payload.blobProperties.getAccountId()) {
                                    String exceptionMsg = "accountid not matching " + " expected " + payload.blobProperties.getAccountId() + " actual " + propertyOutput.getAccountId();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (propertyOutput.getContainerId() != payload.blobProperties.getContainerId()) {
                                    String exceptionMsg = "containerId not matching " + " expected " + payload.blobProperties.getContainerId() + " actual " + propertyOutput.getContainerId();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                if (propertyOutput.isEncrypted() != payload.blobProperties.isEncrypted()) {
                                    String exceptionMsg = "IsEncrypted not matching " + " expected " + payload.blobProperties.isEncrypted() + " actual " + propertyOutput.isEncrypted();
                                    System.out.println(exceptionMsg);
                                    throw new IllegalStateException(exceptionMsg);
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in blobproperty");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException(e);
                            }
                        }
                        // get user metadata
                        ids.clear();
                        ids.add(blobId);
                        partitionRequestInfoList.clear();
                        partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
                        partitionRequestInfoList.add(partitionRequestInfo);
                        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        stream = channel1.receive().getInputStream();
                        resp = GetResponse.readFrom(stream, clusterMap);
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println("Error after get user metadata " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                                if (userMetadataOutput.compareTo(ByteBuffer.wrap(payload.metadata)) != 0) {
                                    throw new IllegalStateException();
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in usermetadatga");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException();
                            }
                        }
                        // get blob
                        ids.clear();
                        ids.add(blobId);
                        partitionRequestInfoList.clear();
                        partitionRequestInfo = new PartitionRequestInfo(ids.get(0).getPartition(), ids);
                        partitionRequestInfoList.add(partitionRequestInfo);
                        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        stream = channel1.receive().getInputStream();
                        resp = GetResponse.readFrom(stream, clusterMap);
                        // System.out.println("response from get " + resp.getError());
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println("Error after get blob " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                                byte[] blobout = new byte[(int) blobData.getSize()];
                                ByteBuf buffer = blobData.content();
                                try {
                                    buffer.readBytes(blobout);
                                } finally {
                                    buffer.release();
                                }
                                if (ByteBuffer.wrap(blobout).compareTo(ByteBuffer.wrap(payload.blob)) != 0) {
                                    throw new IllegalStateException();
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in blobdata");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException();
                            }
                        }
                        // get blob all
                        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
                        channel1.send(getRequest);
                        stream = channel1.receive().getInputStream();
                        resp = GetResponse.readFrom(stream, clusterMap);
                        if (resp.getError() != ServerErrorCode.No_Error) {
                            System.out.println("Error after get blob " + resp.getError());
                            throw new IllegalStateException();
                        } else {
                            try {
                                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), new BlobIdFactory(clusterMap));
                                byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
                                ByteBuf buffer = blobAll.getBlobData().content();
                                try {
                                    buffer.readBytes(blobout);
                                } finally {
                                    buffer.release();
                                }
                                if (ByteBuffer.wrap(blobout).compareTo(ByteBuffer.wrap(payload.blob)) != 0) {
                                    throw new IllegalStateException();
                                }
                                long actualExpiryTimeMs = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs();
                                checkExpiryTimeMatch(payload, actualExpiryTimeMs, "messageinfo in bloball");
                            } catch (MessageFormatException e) {
                                e.printStackTrace();
                                throw new IllegalStateException();
                            }
                        }
                        if (payload.blobProperties.getTimeToLiveInSeconds() != Utils.Infinite_Time) {
                            // ttl update, check and wait for replication
                            ServerTestUtil.updateBlobTtl(channel1, new BlobId(payload.blobId, clusterMap), time.milliseconds());
                            ServerTestUtil.checkTtlUpdateStatus(channel1, clusterMap, new BlobIdFactory(clusterMap), blobId, payload.blob, true, Utils.Infinite_Time);
                            notificationSystem.awaitBlobUpdates(payload.blobId, UpdateType.TTL_UPDATE);
                            BlobProperties old = payload.blobProperties;
                            payload.blobProperties = new BlobProperties(old.getBlobSize(), old.getServiceId(), old.getOwnerId(), old.getContentType(), old.isEncrypted(), Utils.Infinite_Time, old.getCreationTimeInMs(), old.getAccountId(), old.getContainerId(), old.isEncrypted(), old.getExternalAssetTag(), old.getContentEncoding(), old.getFilename());
                        }
                    } catch (Exception e) {
                        if (channel1 != null) {
                            connectionPool.destroyConnection(channel1);
                            channel1 = null;
                        }
                    } finally {
                        if (channel1 != null) {
                            connectionPool.checkInConnection(channel1);
                            channel1 = null;
                        }
                    }
                }
                requestsVerified.incrementAndGet();
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        cancelTest.set(true);
    } finally {
        completedLatch.countDown();
    }
}

Example 28 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class AmbryServerRequestsTest method sendAndVerifyOperationRequest.

/**
 * Sends and verifies that an operation specific request works correctly.
 * @param requestType the type of the request to send.
 * @param ids the partitionIds to send requests for.
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
 *                          response in the constituents rather than the actual response ({@link GetResponse} and
 *                          {@link ReplicaMetadataResponse}).
 * @param forceCheckOpReceived if {@code true}, checks the operation received at the {@link Store} even if
 *                             there is an error expected. Always checks op received if {@code expectedErrorCode} is
 *                             {@link ServerErrorCode#No_Error}. Skips the check otherwise.
 * @param clientIdStr the clientId string to construct request. if null, generate a random string as clientId.
 * @throws InterruptedException
 * @throws IOException
 * @return a list of {@link Response}(s) associated with given partition ids.
 */
private List<Response> sendAndVerifyOperationRequest(RequestOrResponseType requestType, List<? extends PartitionId> ids, ServerErrorCode expectedErrorCode, Boolean forceCheckOpReceived, String clientIdStr) throws InterruptedException, IOException {
    List<Response> responses = new ArrayList<>();
    for (PartitionId id : ids) {
        int correlationId = TestUtils.RANDOM.nextInt();
        String clientId = clientIdStr == null ? TestUtils.getRandomString(10) : clientIdStr;
        BlobId originalBlobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), id, false, BlobId.BlobDataType.DATACHUNK);
        BlobId convertedBlobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.CRAFTED, ClusterMap.UNKNOWN_DATACENTER_ID, originalBlobId.getAccountId(), originalBlobId.getContainerId(), id, false, BlobId.BlobDataType.DATACHUNK);
        conversionMap.put(originalBlobId, convertedBlobId);
        validKeysInStore.add(convertedBlobId);
        RequestOrResponse request;
        switch(requestType) {
            case PutRequest:
                BlobProperties properties = new BlobProperties(0, "serviceId", originalBlobId.getAccountId(), originalBlobId.getAccountId(), false);
                request = new PutRequest(correlationId, clientId, originalBlobId, properties, ByteBuffer.allocate(0), Unpooled.wrappedBuffer(ByteBuffer.allocate(0)), 0, BlobType.DataBlob, null);
                break;
            case DeleteRequest:
                request = new DeleteRequest(correlationId, clientId, originalBlobId, SystemTime.getInstance().milliseconds());
                break;
            case UndeleteRequest:
                request = new UndeleteRequest(correlationId, clientId, originalBlobId, SystemTime.getInstance().milliseconds());
                break;
            case GetRequest:
                PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(id, Collections.singletonList(originalBlobId));
                request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
                break;
            case ReplicaMetadataRequest:
                ReplicaMetadataRequestInfo rRequestInfo = new ReplicaMetadataRequestInfo(id, findTokenHelper.getFindTokenFactoryFromReplicaType(ReplicaType.DISK_BACKED).getNewFindToken(), "localhost", "/tmp", ReplicaType.DISK_BACKED, replicationConfig.replicaMetadataRequestVersion);
                request = new ReplicaMetadataRequest(correlationId, clientId, Collections.singletonList(rRequestInfo), Long.MAX_VALUE, replicationConfig.replicaMetadataRequestVersion);
                break;
            case TtlUpdateRequest:
                request = new TtlUpdateRequest(correlationId, clientId, originalBlobId, Utils.Infinite_Time, SystemTime.getInstance().milliseconds());
                break;
            default:
                throw new IllegalArgumentException(requestType + " not supported by this function");
        }
        responses.add(sendAndVerifyOperationRequest(request, expectedErrorCode, forceCheckOpReceived));
    }
    return responses;
}

Example 29 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class ServerHardDeleteTest method getAndVerify.

/**
 * Fetches the Blob(for all MessageFormatFlags) and verifies the content
 * @param channel the {@link BlockingChannel} to use to send and receive data
 * @param blobsCount the total number of blobs that needs to be verified against
 * @throws Exception
 */
void getAndVerify(ConnectedChannel channel, int blobsCount) throws Exception {
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<>();
    ArrayList<BlobId> ids = new ArrayList<>();
    for (int i = 0; i < blobsCount; i++) {
        ids.add(blobIdList.get(i));
    }
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIdList.get(0).getPartition(), ids);
    partitionRequestInfoList.add(partitionRequestInfo);
    ArrayList<MessageFormatFlags> flags = new ArrayList<>();
    flags.add(MessageFormatFlags.BlobProperties);
    flags.add(MessageFormatFlags.BlobUserMetadata);
    flags.add(MessageFormatFlags.Blob);
    for (MessageFormatFlags flag : flags) {
        GetRequest getRequest = new GetRequest(1, "clientid2", flag, partitionRequestInfoList, GetOption.Include_All);
        GetResponse resp = GetResponse.readFrom(channel.sendAndReceive(getRequest).getInputStream(), mockClusterMap);
        if (flag == MessageFormatFlags.BlobProperties) {
            for (int i = 0; i < blobsCount; i++) {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                Assert.assertEquals(properties.get(i).getBlobSize(), propertyOutput.getBlobSize());
                Assert.assertEquals("serviceid1", propertyOutput.getServiceId());
                Assert.assertEquals("AccountId mismatch", properties.get(i).getAccountId(), propertyOutput.getAccountId());
                Assert.assertEquals("ContainerId mismatch", properties.get(i).getContainerId(), propertyOutput.getContainerId());
            }
        } else if (flag == MessageFormatFlags.BlobUserMetadata) {
            for (int i = 0; i < blobsCount; i++) {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                Assert.assertArrayEquals(userMetadataOutput.array(), usermetadata.get(i));
            }
        } else if (flag == MessageFormatFlags.Blob) {
            for (int i = 0; i < blobsCount; i++) {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                Assert.assertEquals(properties.get(i).getBlobSize(), blobData.getSize());
                byte[] dataOutput = new byte[(int) blobData.getSize()];
                ByteBuf buffer = blobData.content();
                try {
                    buffer.readBytes(dataOutput);
                } finally {
                    buffer.release();
                }
                Assert.assertArrayEquals(dataOutput, data.get(i));
            }
        } else {
            throw new IllegalArgumentException("Unrecognized message format flags " + flags);
        }
    }
}

Example 30 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class ServerHardDeleteTest method endToEndTestHardDeletes.

/**
 * Tests the hard delete functionality.
 * <p>
 * This test does the following:
 * 1. Makes 6 puts, waits for notification.
 * 2. Makes 2 deletes, waits for notification.
 * 3. Waits for hard deletes to catch up to the expected token value.
 * 4. Verifies that the two records that are deleted are zeroed out by hard deletes.
 * 5. Makes 3 more puts, waits for notification.
 * 6. Makes 3 deletes - 2 of records from the initial set of puts, and 1 from the new set.
 * 7. Waits for hard deletes to catch up again to the expected token value.
 * 8. Verifies that the three records that are deleted are zeroed out by hard deletes.
 *
 * @throws Exception
 */
@Test
public void endToEndTestHardDeletes() throws Exception {
    DataNodeId dataNodeId = mockClusterMap.getDataNodeIds().get(0);
    encryptionKey = new ArrayList<>(9);
    usermetadata = new ArrayList<>(9);
    data = new ArrayList<>(9);
    Random random = new Random();
    for (int i = 0; i < 9; i++) {
        if (i % 2 == 0) {
            encryptionKey.add(new byte[100]);
            random.nextBytes(encryptionKey.get(i));
        } else {
            encryptionKey.add(null);
        }
        usermetadata.add(new byte[1000 + i]);
        data.add(new byte[31870 + i]);
        random.nextBytes(usermetadata.get(i));
        random.nextBytes(data.get(i));
    }
    properties = new ArrayList<>(9);
    properties.add(new BlobProperties(31870, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31871, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false));
    properties.add(new BlobProperties(31872, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31873, "serviceid1", "ownerid", "jpeg", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null));
    properties.add(new BlobProperties(31874, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31875, "serviceid1", "ownerid", "jpeg", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null));
    properties.add(new BlobProperties(31876, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31877, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false));
    properties.add(new BlobProperties(31878, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    List<PartitionId> partitionIds = mockClusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
    PartitionId chosenPartition = partitionIds.get(0);
    blobIdList = new ArrayList<>(9);
    for (int i = 0; i < 9; i++) {
        blobIdList.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, mockClusterMap.getLocalDatacenterId(), properties.get(i).getAccountId(), properties.get(i).getContainerId(), chosenPartition, false, BlobId.BlobDataType.DATACHUNK));
    }
    ConnectedChannel channel = ServerTestUtil.getBlockingChannelBasedOnPortType(new Port(dataNodeId.getPort(), PortType.PLAINTEXT), "localhost", null, null);
    channel.connect();
    for (int i = 0; i < 6; i++) {
        // blob 3 and 5 are expired among these
        putBlob(blobIdList.get(i), properties.get(i), encryptionKey.get(i), usermetadata.get(i), data.get(i), channel);
    }
    notificationSystem.awaitBlobCreations(blobIdList.get(0).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(1).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(2).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(4).getID());
    // delete blob 1
    deleteBlob(blobIdList.get(1), channel);
    zeroOutBlobContent(1);
    // delete blob 4
    deleteBlob(blobIdList.get(4), channel);
    zeroOutBlobContent(4);
    notificationSystem.awaitBlobDeletions(blobIdList.get(1).getID());
    notificationSystem.awaitBlobDeletions(blobIdList.get(4).getID());
    time.sleep(TimeUnit.DAYS.toMillis(7));
    // For each future change to this offset, add to this variable and write an explanation of why the number changed.
    // old value: 198728. Increased by 4 to 198732 because the format for delete record went from 2 to 3 which adds
    // 4 bytes (two shorts) extra. The last record is a delete record so its extra 4 bytes are not (yet) added
    // 
    // Add 14 here when changing message header version to 3, since the message header version went from 2 to 3 and adds
    // a short to every record, which include 6 puts and 1 delete. (last delete is not included).
    // old value is 198732 + 14. Increased by 48 when adding two fields(4 BYTE CRC for each field) in blobProperty when putBlob.
    // There are 6 * (4 + 4). 6 stands for the times for putBlob, 4 stands for 4 extra blobProperty Bytes for each field.
    int expectedTokenValueT1 = 198732 + 14 + 48;
    ensureCleanupTokenCatchesUp(chosenPartition.getReplicaIds().get(0).getReplicaPath(), mockClusterMap, expectedTokenValueT1);
    getAndVerify(channel, 6);
    // put blob 6
    putBlob(blobIdList.get(6), properties.get(6), encryptionKey.get(6), usermetadata.get(6), data.get(6), channel);
    // put blob 7
    putBlob(blobIdList.get(7), properties.get(7), encryptionKey.get(7), usermetadata.get(7), data.get(7), channel);
    // put blob 8
    putBlob(blobIdList.get(8), properties.get(8), encryptionKey.get(8), usermetadata.get(8), data.get(8), channel);
    notificationSystem.awaitBlobCreations(blobIdList.get(6).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(7).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(8).getID());
    // Do more deletes
    // delete blob 3 that is expired.
    deleteBlob(blobIdList.get(3), channel);
    zeroOutBlobContent(3);
    // delete blob 0, will undelete it later, so don't zero out the content
    deleteBlob(blobIdList.get(0), channel);
    // delete blob 6.
    deleteBlob(blobIdList.get(6), channel);
    zeroOutBlobContent(6);
    notificationSystem.awaitBlobDeletions(blobIdList.get(0).getID());
    notificationSystem.awaitBlobDeletions(blobIdList.get(6).getID());
    undeleteBlob(blobIdList.get(0), channel);
    notificationSystem.awaitBlobUndeletes(blobIdList.get(0).getID());
    time.sleep(TimeUnit.DAYS.toMillis(1));
    // For each future change to this offset, add to this variable and write an explanation of why the number changed.
    int expectedTokenValueT2 = 298416 + 98 + 28 + 72;
    // old value: 298400. Increased by 16 (4 * 4) to 298416 because the format for delete record went from 2 to 3 which
    // adds 4 bytes (two shorts) extra. The last record is a delete record so its extra 4 bytes are not added
    // 
    // old value 298416. Increased by 98. The end offset is now a journal-based offset, so the offset is not inclusive.
    // It points to the last record in the journal. Before adding an undelete record, the last record in journal is the
    // delete record for blob 6, now it's undelete for blob 0. Since a delete record is 98 bytes, so increase 98 bytes.
    // 
    // old value is 298416 + 98. Increased by 28 when changing the message header version from 2 to 3, which adds a short
    // to all the records, which includes 9 puts and 5 deletes and 1 undelete. Undelete is not include since it's the last
    // record.
    // old value is 298416 + 98 + 28. Increased by 72 when adding two fields(4 BYTE CRC for each field) in blobProperty when putBlob.
    // There are 9 * (4 + 4). 9 stands for the times for putBlob, 4 stands for 4 extra blobProperty Bytes.
    ensureCleanupTokenCatchesUp(chosenPartition.getReplicaIds().get(0).getReplicaPath(), mockClusterMap, expectedTokenValueT2);
    getAndVerify(channel, 9);
}