Examples with BlobStoreControlAdminRequest com.github.ambry.protocol.BlobStoreControlAdminRequest used on opensource projects

Example 1 with BlobStoreControlAdminRequest

use of com.github.ambry.protocol.BlobStoreControlAdminRequest in project ambry by linkedin.

the class AmbryRequestsTest method stopBlobStoreFailureTest.

/**
 * Tests for the response received on a {@link BlobStoreControlAdminRequest} for different failure cases
 * @throws InterruptedException
 * @throws IOException
 */
@Test
public void stopBlobStoreFailureTest() throws InterruptedException, IOException {
    List<? extends PartitionId> partitionIds = clusterMap.getAllPartitionIds();
    PartitionId id = partitionIds.get(0);
    int correlationId = TestUtils.RANDOM.nextInt();
    String clientId = UtilsTest.getRandomString(10);
    short numReplicasCaughtUpPerPartition = -1;
    // test invalid numReplicasCaughtUpPerPartition
    AdminRequest adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    BlobStoreControlAdminRequest blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    Response response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Bad_Request);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    // test partition unknown
    numReplicasCaughtUpPerPartition = 3;
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, null, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Partition_Unknown);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    // test validate request failure
    storageManager.returnNullStore = true;
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Disk_Unavailable);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    storageManager.returnNullStore = false;
    // test disable compaction failure
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    storageManager.returnValueOfDisablingCompaction = false;
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Unknown_Error);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    storageManager.returnValueOfDisablingCompaction = true;
    // test disable compaction with runtime exception
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    storageManager.exceptionToThrowOnDisablingCompaction = new IllegalStateException();
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Unknown_Error);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    storageManager.exceptionToThrowOnDisablingCompaction = null;
    // test disable replication failure
    replicationManager.reset();
    replicationManager.controlReplicationReturnVal = false;
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Unknown_Error);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    // test peers catchup failure
    replicationManager.reset();
    replicationManager.controlReplicationReturnVal = true;
    // all replicas of this partition > acceptableLag
    generateLagOverrides(1, 1);
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Retry_After_Backoff);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    // test shutdown BlobStore failure
    replicationManager.reset();
    replicationManager.controlReplicationReturnVal = true;
    storageManager.returnValueOfShutdownBlobStore = false;
    generateLagOverrides(0, 0);
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Unknown_Error);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    // test shutdown BlobStore with runtime exception
    storageManager.exceptionToThrowOnShuttingdownBlobStore = new IllegalStateException();
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    blobStoreControlAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    response = sendRequestGetResponse(blobStoreControlAdminRequest, ServerErrorCode.Unknown_Error);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    storageManager.exceptionToThrowOnShuttingdownBlobStore = null;
}

Example 2 with BlobStoreControlAdminRequest

use of com.github.ambry.protocol.BlobStoreControlAdminRequest in project ambry by linkedin.

the class ServerTestUtil method endToEndTest.

static void endToEndTest(Port targetPort, String routerDatacenter, MockCluster cluster, SSLConfig clientSSLConfig, SSLSocketFactory clientSSLSocketFactory, Properties routerProps, boolean testEncryption) {
    try {
        MockClusterMap clusterMap = cluster.getClusterMap();
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        byte[] userMetadata = new byte[1000];
        byte[] data = new byte[31870];
        byte[] encryptionKey = new byte[100];
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        BlobProperties properties = new BlobProperties(31870, "serviceid1", accountId, containerId, testEncryption, cluster.time.milliseconds());
        TestUtils.RANDOM.nextBytes(userMetadata);
        TestUtils.RANDOM.nextBytes(data);
        if (testEncryption) {
            TestUtils.RANDOM.nextBytes(encryptionKey);
        }
        List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
        short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
        BlobId blobId1 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId2 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId3 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId4 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId5 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        // put blob 1
        PutRequest putRequest = new PutRequest(1, "client1", blobId1, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        ConnectedChannel channel = getBlockingChannelBasedOnPortType(targetPort, "localhost", clientSSLSocketFactory, clientSSLConfig);
        channel.connect();
        DataInputStream putResponseStream = channel.sendAndReceive(putRequest).getInputStream();
        PutResponse response = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response.getError());
        // put blob 2 with an expiry time and apply TTL update later
        BlobProperties propertiesForTtlUpdate = new BlobProperties(31870, "serviceid1", "ownerid", "image/png", false, TestUtils.TTL_SECS, cluster.time.milliseconds(), accountId, containerId, testEncryption, null, null, null);
        long ttlUpdateBlobExpiryTimeMs = getExpiryTimeMs(propertiesForTtlUpdate);
        PutRequest putRequest2 = new PutRequest(1, "client1", blobId2, propertiesForTtlUpdate, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest2).getInputStream();
        PutResponse response2 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 3
        PutRequest putRequest3 = new PutRequest(1, "client1", blobId3, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest3).getInputStream();
        PutResponse response3 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // put blob 4 that is expired
        BlobProperties propertiesExpired = new BlobProperties(31870, "serviceid1", "ownerid", "jpeg", false, 0, cluster.time.milliseconds(), accountId, containerId, testEncryption, null, null, null);
        PutRequest putRequest4 = new PutRequest(1, "client1", blobId4, propertiesExpired, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest4).getInputStream();
        PutResponse response4 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response4.getError());
        cluster.time.sleep(10000);
        // get blob properties
        ArrayList<BlobId> ids = new ArrayList<>();
        MockPartitionId partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(blobId1);
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        DataInputStream stream = channel.sendAndReceive(getRequest1).getInputStream();
        GetResponse resp1 = GetResponse.readFrom(stream, clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            releaseNettyBufUnderneathStream(stream);
        } catch (MessageFormatException e) {
            fail();
        }
        // get blob properties with expired flag set
        ids = new ArrayList<BlobId>();
        partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.Include_Expired_Blobs);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            releaseNettyBufUnderneathStream(stream);
        } catch (MessageFormatException e) {
            fail();
        }
        // get blob properties for expired blob
        // 1. With no flag
        ArrayList<BlobId> idsExpired = new ArrayList<>();
        MockPartitionId partitionExpired = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        idsExpired.add(blobId4);
        ArrayList<PartitionRequestInfo> partitionRequestInfoListExpired = new ArrayList<>();
        PartitionRequestInfo partitionRequestInfoExpired = new PartitionRequestInfo(partitionExpired, idsExpired);
        partitionRequestInfoListExpired.add(partitionRequestInfoExpired);
        GetRequest getRequestExpired = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoListExpired, GetOption.None);
        DataInputStream streamExpired = channel.sendAndReceive(getRequestExpired).getInputStream();
        GetResponse respExpired = GetResponse.readFrom(streamExpired, clusterMap);
        assertEquals(ServerErrorCode.Blob_Expired, respExpired.getPartitionResponseInfoList().get(0).getErrorCode());
        releaseNettyBufUnderneathStream(streamExpired);
        // 2. With Include_Expired flag
        idsExpired = new ArrayList<>();
        partitionExpired = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        idsExpired.add(blobId4);
        partitionRequestInfoListExpired = new ArrayList<>();
        partitionRequestInfoExpired = new PartitionRequestInfo(partitionExpired, idsExpired);
        partitionRequestInfoListExpired.add(partitionRequestInfoExpired);
        getRequestExpired = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoListExpired, GetOption.Include_Expired_Blobs);
        streamExpired = channel.sendAndReceive(getRequestExpired).getInputStream();
        respExpired = GetResponse.readFrom(streamExpired, clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(respExpired.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("ownerid", propertyOutput.getOwnerId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            releaseNettyBufUnderneathStream(streamExpired);
        } catch (MessageFormatException e) {
            fail();
        }
        // get user metadata
        GetRequest getRequest2 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest2).getInputStream();
        GetResponse resp2 = GetResponse.readFrom(stream, clusterMap);
        try {
            ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp2.getInputStream());
            assertArrayEquals(userMetadata, userMetadataOutput.array());
            if (testEncryption) {
                assertNotNull("MessageMetadata should not have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
            } else {
                assertNull("MessageMetadata should have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            }
            releaseNettyBufUnderneathStream(stream);
        } catch (MessageFormatException e) {
            fail();
        }
        // get blob info
        GetRequest getRequest3 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobInfo, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest3).getInputStream();
        GetResponse resp3 = GetResponse.readFrom(stream, clusterMap);
        InputStream responseStream = resp3.getInputStream();
        // verify blob properties.
        BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(responseStream);
        assertEquals(31870, propertyOutput.getBlobSize());
        assertEquals("serviceid1", propertyOutput.getServiceId());
        assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
        assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
        // verify user metadata
        ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(responseStream);
        assertArrayEquals(userMetadata, userMetadataOutput.array());
        if (testEncryption) {
            assertNotNull("MessageMetadata should not have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
        } else {
            assertNull("MessageMetadata should have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob all
        GetRequest getRequest4 = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest4).getInputStream();
        GetResponse resp4 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp4.getInputStream();
        BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        // verify content
        assertArrayEquals("Content mismatch.", data, actualBlobData);
        if (testEncryption) {
            assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
            assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
        } else {
            assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
        }
        releaseNettyBufUnderneathStream(stream);
        // encryptionKey in this test doesn't have any relation to the content. Both are random bytes for test purposes.
        if (!testEncryption) {
            // Use router to get the blob
            Properties routerProperties = getRouterProps(routerDatacenter);
            routerProperties.putAll(routerProps);
            VerifiableProperties routerVerifiableProps = new VerifiableProperties(routerProperties);
            AccountService accountService = new InMemAccountService(false, true);
            Router router = new NonBlockingRouterFactory(routerVerifiableProps, clusterMap, new MockNotificationSystem(clusterMap), getSSLFactoryIfRequired(routerVerifiableProps), accountService).getRouter();
            checkBlobId(router, blobId1, data);
            router.close();
        }
        checkTtlUpdateStatus(channel, clusterMap, blobIdFactory, blobId2, data, false, ttlUpdateBlobExpiryTimeMs);
        updateBlobTtl(channel, blobId2, cluster.time.milliseconds());
        checkTtlUpdateStatus(channel, clusterMap, blobIdFactory, blobId2, data, true, Utils.Infinite_Time);
        // fetch blob that does not exist
        // get blob properties
        ids = new ArrayList<>();
        partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partition, false, BlobId.BlobDataType.DATACHUNK));
        partitionRequestInfoList.clear();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest5 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest5).getInputStream();
        GetResponse resp5 = GetResponse.readFrom(stream, clusterMap);
        assertEquals(ServerErrorCode.Blob_Not_Found, resp5.getPartitionResponseInfoList().get(0).getErrorCode());
        releaseNettyBufUnderneathStream(stream);
        // stop the store via AdminRequest
        System.out.println("Begin to stop a BlobStore");
        AdminRequest adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionIds.get(0), 1, "clientid2");
        BlobStoreControlAdminRequest controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StopStore, adminRequest);
        stream = channel.sendAndReceive(controlRequest).getInputStream();
        AdminResponse adminResponse = AdminResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Stop store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
        // put a blob on a stopped store, which should fail
        putRequest = new PutRequest(1, "client1", blobId5, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest).getInputStream();
        response = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals("Put blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, response.getError());
        // get a blob properties on a stopped store, which should fail
        ids = new ArrayList<>();
        partition = (MockPartitionId) blobId1.getPartition();
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        assertEquals("Get blob properties on stopped store should fail", ServerErrorCode.Replica_Unavailable, resp1.getPartitionResponseInfoList().get(0).getErrorCode());
        releaseNettyBufUnderneathStream(stream);
        // delete a blob on a stopped store, which should fail
        DeleteRequest deleteRequest = new DeleteRequest(1, "deleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(deleteRequest).getInputStream();
        DeleteResponse deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Delete blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, deleteResponse.getError());
        // start the store via AdminRequest
        System.out.println("Begin to restart the BlobStore");
        adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionIds.get(0), 1, "clientid2");
        controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StartStore, adminRequest);
        stream = channel.sendAndReceive(controlRequest).getInputStream();
        adminResponse = AdminResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Start store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
        List<? extends ReplicaId> replicaIds = partitionIds.get(0).getReplicaIds();
        for (ReplicaId replicaId : replicaIds) {
            // forcibly mark replicas and disks as up.
            MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
            mockReplicaId.markReplicaDownStatus(false);
            ((MockDiskId) mockReplicaId.getDiskId()).setDiskState(HardwareState.AVAILABLE, false);
        }
        // put a blob on a restarted store , which should succeed
        PutRequest putRequest5 = new PutRequest(1, "client1", blobId5, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest5).getInputStream();
        PutResponse response5 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals("Put blob on restarted store should succeed", ServerErrorCode.No_Error, response5.getError());
        // get a blob on a restarted store , which should succeed
        ids = new ArrayList<>();
        PartitionId partitionId = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partitionId, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp1.getInputStream();
        blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        assertArrayEquals("Content mismatch.", data, actualBlobData);
        releaseNettyBufUnderneathStream(stream);
        // undelete a not-deleted blob should return fail
        UndeleteRequest undeleteRequest = new UndeleteRequest(1, "undeleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(undeleteRequest).getInputStream();
        UndeleteResponse undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Undelete blob should succeed", ServerErrorCode.Blob_Not_Deleted, undeleteResponse.getError());
        // delete a blob on a restarted store , which should succeed
        deleteRequest = new DeleteRequest(1, "deleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(deleteRequest).getInputStream();
        deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Delete blob on restarted store should succeed", ServerErrorCode.No_Error, deleteResponse.getError());
        // undelete a deleted blob, which should succeed
        undeleteRequest = new UndeleteRequest(2, "undeleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(undeleteRequest).getInputStream();
        undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Undelete blob should succeed", ServerErrorCode.No_Error, undeleteResponse.getError());
        assertEquals("Undelete life version mismatch", undeleteResponse.getLifeVersion(), (short) 1);
        // undelete an already undeleted blob, which should fail
        undeleteRequest = new UndeleteRequest(3, "undeleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(undeleteRequest).getInputStream();
        undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Undelete blob should fail", ServerErrorCode.Blob_Already_Undeleted, undeleteResponse.getError());
        assertEquals("LifeVersion Mismatch", (short) 1, undeleteResponse.getLifeVersion());
        // get an undeleted blob, which should succeed
        getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp1.getInputStream();
        blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        assertArrayEquals("Content mismatch", data, actualBlobData);
        releaseNettyBufUnderneathStream(stream);
        // Bounce servers to make them read the persisted token file.
        cluster.stopServers();
        cluster.reinitServers();
        channel.disconnect();
        channel.connect();
        // get an undeleted blob after restart, which should succeed
        getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp1.getInputStream();
        blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        assertArrayEquals("Content mismatch", data, actualBlobData);
        releaseNettyBufUnderneathStream(stream);
        channel.disconnect();
    } catch (Exception e) {
        e.printStackTrace();
        assertNull(e);
    } finally {
        List<? extends ReplicaId> replicaIds = cluster.getClusterMap().getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0).getReplicaIds();
        for (ReplicaId replicaId : replicaIds) {
            MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
            ((MockDiskId) mockReplicaId.getDiskId()).setDiskState(HardwareState.AVAILABLE, true);
        }
    }
}

Example 3 with BlobStoreControlAdminRequest

use of com.github.ambry.protocol.BlobStoreControlAdminRequest in project ambry by linkedin.

the class AmbryRequests method handleBlobStoreControlRequest.

/**
 * Handles {@link com.github.ambry.protocol.AdminRequestOrResponseType#BlobStoreControl}.
 * @param requestStream the serialized bytes of the request.
 * @param adminRequest the {@link AdminRequest} received.
 * @return the {@link AdminResponse} to the request.
 * @throws IOException if there is any I/O error reading from the {@code requestStream}.
 */
private AdminResponse handleBlobStoreControlRequest(DataInputStream requestStream, AdminRequest adminRequest) throws IOException {
    ServerErrorCode error;
    BlobStoreControlAdminRequest blobStoreControlAdminRequest = BlobStoreControlAdminRequest.readFrom(requestStream, adminRequest);
    if (blobStoreControlAdminRequest.getNumReplicasCaughtUpPerPartition() > 0) {
        PartitionId partitionId = blobStoreControlAdminRequest.getPartitionId();
        if (partitionId != null) {
            error = validateRequest(partitionId, RequestOrResponseType.AdminRequest);
            if (error.equals(ServerErrorCode.No_Error)) {
                if (blobStoreControlAdminRequest.shouldEnable()) {
                    // TODO: start BlobStore properly
                    error = ServerErrorCode.Temporarily_Disabled;
                } else {
                    Collection<PartitionId> partitionIds = Collections.singletonList(partitionId);
                    if (storageManager.disableCompactionForBlobStore(partitionId)) {
                        controlRequestForPartitions(RequestOrResponseType.PutRequest, partitionIds, false);
                        controlRequestForPartitions(RequestOrResponseType.DeleteRequest, partitionIds, false);
                        if (replicationManager.controlReplicationForPartitions(partitionIds, Collections.<String>emptyList(), false)) {
                            if (isRemoteLagLesserOrEqual(partitionIds, 0, blobStoreControlAdminRequest.getNumReplicasCaughtUpPerPartition())) {
                                controlRequestForPartitions(RequestOrResponseType.GetRequest, partitionIds, false);
                                controlRequestForPartitions(RequestOrResponseType.ReplicaMetadataRequest, partitionIds, false);
                                // Shutdown the BlobStore completely
                                if (storageManager.shutdownBlobStore(partitionId)) {
                                    error = ServerErrorCode.No_Error;
                                    logger.info("store shutdown for partition: {}", partitionId);
                                } else {
                                    error = ServerErrorCode.Unknown_Error;
                                    logger.error("Shutting down BlobStore fails on {}", partitionId);
                                }
                            } else {
                                error = ServerErrorCode.Retry_After_Backoff;
                                logger.debug("Catchup not done on {}", partitionIds);
                            }
                        } else {
                            error = ServerErrorCode.Unknown_Error;
                            logger.error("Could not disable replication on {}", partitionIds);
                        }
                    } else {
                        error = ServerErrorCode.Unknown_Error;
                        logger.error("Disable compaction on given BlobStore failed for {}", partitionId);
                    }
                }
            } else {
                logger.debug("Validate request fails for {} with error code {}", partitionId, error);
            }
        } else {
            error = ServerErrorCode.Partition_Unknown;
            logger.debug("The partition Id should not be null.");
        }
    } else {
        error = ServerErrorCode.Bad_Request;
        logger.debug("The number of replicas to catch up should not be less or equal to zero {}", blobStoreControlAdminRequest.getNumReplicasCaughtUpPerPartition());
    }
    return new AdminResponse(adminRequest.getCorrelationId(), adminRequest.getClientId(), error);
}

Example 4 with BlobStoreControlAdminRequest

use of com.github.ambry.protocol.BlobStoreControlAdminRequest in project ambry by linkedin.

the class AmbryRequestsTest method stopBlobStoreSuccessTest.

/**
 * Tests for the response received on a {@link BlobStoreControlAdminRequest} for successful case
 * @throws InterruptedException
 * @throws IOException
 */
@Test
public void stopBlobStoreSuccessTest() throws InterruptedException, IOException {
    List<? extends PartitionId> partitionIds = clusterMap.getAllPartitionIds();
    PartitionId id = partitionIds.get(0);
    List<? extends ReplicaId> replicaIds = id.getReplicaIds();
    assertTrue("This test needs more than one replica for the first partition to work", replicaIds.size() > 1);
    int correlationId = TestUtils.RANDOM.nextInt();
    String clientId = UtilsTest.getRandomString(10);
    long acceptableLagInBytes = 0;
    short numReplicasCaughtUpPerPartition = 3;
    replicationManager.reset();
    replicationManager.controlReplicationReturnVal = true;
    generateLagOverrides(0, acceptableLagInBytes);
    // stop BlobStore
    AdminRequest adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, id, correlationId, clientId);
    BlobStoreControlAdminRequest stopBlobStoreAdminRequest = new BlobStoreControlAdminRequest(numReplicasCaughtUpPerPartition, false, adminRequest);
    Response response = sendRequestGetResponse(stopBlobStoreAdminRequest, ServerErrorCode.No_Error);
    assertTrue("Response not of type AdminResponse", response instanceof AdminResponse);
    // test APIs are called in the process of stopping BlobStore
    assertEquals("Partition disabled for compaction not as expected", id, storageManager.compactionDisabledPartitionId);
    assertEquals("Origins list should be empty", true, replicationManager.originsVal.isEmpty());
    assertEquals("Replication on given BlobStore should be disabled", false, replicationManager.enableVal);
    assertEquals("Partition shutdown not as expected", id, storageManager.shutdownPartitionId);
}

Example 5 with BlobStoreControlAdminRequest

use of com.github.ambry.protocol.BlobStoreControlAdminRequest in project ambry by linkedin.

the class ServerTestUtil method endToEndReplicationWithMultiNodeMultiPartitionMultiDCTest.

static void endToEndReplicationWithMultiNodeMultiPartitionMultiDCTest(String sourceDatacenter, String sslEnabledDatacenters, PortType portType, MockCluster cluster, MockNotificationSystem notificationSystem, Properties routerProps) throws Exception {
    Properties props = new Properties();
    props.setProperty("router.hostname", "localhost");
    props.setProperty("router.datacenter.name", sourceDatacenter);
    props.setProperty("router.put.request.parallelism", "1");
    props.setProperty("router.put.success.target", "1");
    props.setProperty("clustermap.cluster.name", "test");
    props.setProperty("clustermap.datacenter.name", sourceDatacenter);
    props.setProperty("clustermap.host.name", "localhost");
    props.setProperty("kms.default.container.key", TestUtils.getRandomKey(32));
    props.putAll(routerProps);
    VerifiableProperties verifiableProperties = new VerifiableProperties(props);
    AccountService accountService = new InMemAccountService(false, true);
    Router router = new NonBlockingRouterFactory(verifiableProperties, cluster.getClusterMap(), notificationSystem, getSSLFactoryIfRequired(verifiableProperties), accountService).getRouter();
    int numberOfRequestsToSend = 15;
    int numberOfVerifierThreads = 3;
    final LinkedBlockingQueue<Payload> payloadQueue = new LinkedBlockingQueue<Payload>();
    final AtomicReference<Exception> exceptionRef = new AtomicReference<>(null);
    final CountDownLatch callbackLatch = new CountDownLatch(numberOfRequestsToSend);
    List<Future<String>> putFutures = new ArrayList<>(numberOfRequestsToSend);
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    for (int i = 0; i < numberOfRequestsToSend; i++) {
        int size = new Random().nextInt(5000);
        final BlobProperties properties = new BlobProperties(size, "service1", "owner id check", "image/jpeg", false, TestUtils.TTL_SECS, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
        final byte[] metadata = new byte[new Random().nextInt(1000)];
        final byte[] blob = new byte[size];
        TestUtils.RANDOM.nextBytes(metadata);
        TestUtils.RANDOM.nextBytes(blob);
        Future<String> future = router.putBlob(properties, metadata, new ByteBufferReadableStreamChannel(ByteBuffer.wrap(blob)), new PutBlobOptionsBuilder().build(), new Callback<String>() {

            @Override
            public void onCompletion(String result, Exception exception) {
                if (exception == null) {
                    payloadQueue.add(new Payload(properties, metadata, blob, result));
                } else {
                    exceptionRef.set(exception);
                }
                callbackLatch.countDown();
            }
        }, QUOTA_CHARGE_EVENT_LISTENER);
        putFutures.add(future);
    }
    for (Future<String> future : putFutures) {
        future.get(20, TimeUnit.SECONDS);
    }
    assertTrue("Did not receive all callbacks in time", callbackLatch.await(1, TimeUnit.SECONDS));
    if (exceptionRef.get() != null) {
        throw exceptionRef.get();
    }
    // put away for future use
    Payload payload1 = payloadQueue.peek();
    MockClusterMap clusterMap = cluster.getClusterMap();
    BlobId blobId1 = new BlobId(payload1.blobId, clusterMap);
    assertEquals("Did not put expected number of blobs", numberOfRequestsToSend, payloadQueue.size());
    Properties sslProps = new Properties();
    sslProps.putAll(routerProps);
    sslProps.setProperty("clustermap.ssl.enabled.datacenters", sslEnabledDatacenters);
    sslProps.setProperty("clustermap.cluster.name", "test");
    sslProps.setProperty("clustermap.datacenter.name", sourceDatacenter);
    sslProps.setProperty("clustermap.host.name", "localhost");
    sslProps.setProperty("connectionpool.read.timeout.ms", "15000");
    VerifiableProperties vProps = new VerifiableProperties(sslProps);
    ConnectionPool connectionPool = new BlockingChannelConnectionPool(new ConnectionPoolConfig(vProps), new SSLConfig(vProps), new ClusterMapConfig(vProps), new MetricRegistry());
    CountDownLatch verifierLatch = new CountDownLatch(numberOfVerifierThreads);
    AtomicInteger totalRequests = new AtomicInteger(numberOfRequestsToSend);
    AtomicInteger verifiedRequests = new AtomicInteger(0);
    AtomicBoolean cancelTest = new AtomicBoolean(false);
    for (int i = 0; i < numberOfVerifierThreads; i++) {
        Thread thread = new Thread(new Verifier(payloadQueue, verifierLatch, totalRequests, verifiedRequests, cluster.getClusterMap(), cancelTest, portType, connectionPool, notificationSystem, cluster.time));
        thread.start();
    }
    assertTrue("Did not verify in 2 minutes", verifierLatch.await(2, TimeUnit.MINUTES));
    assertEquals(totalRequests.get(), verifiedRequests.get());
    BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
    MockDataNodeId dataNodeId = clusterMap.getDataNodes().get(0);
    Port port = new Port(portType == PortType.PLAINTEXT ? dataNodeId.getPort() : dataNodeId.getSSLPort(), portType);
    ConnectedChannel channel = connectionPool.checkOutConnection("localhost", port, 10000);
    PartitionId partitionId = blobId1.getPartition();
    // stop the store via AdminRequest
    System.out.println("Begin to stop a BlobStore");
    AdminRequest adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionId, 1, "clientid2");
    BlobStoreControlAdminRequest controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StopStore, adminRequest);
    DataInputStream stream = channel.sendAndReceive(controlRequest).getInputStream();
    AdminResponse adminResponse = AdminResponse.readFrom(stream);
    releaseNettyBufUnderneathStream(stream);
    assertEquals("Stop store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
    // put a blob on a stopped store, which should fail
    byte[] userMetadata = new byte[1000];
    byte[] data = new byte[3187];
    BlobProperties properties = new BlobProperties(3187, "serviceid1", accountId, containerId, false, cluster.time.milliseconds());
    BlobId blobId2 = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), accountId, containerId, partitionId, false, BlobId.BlobDataType.DATACHUNK);
    PutRequest putRequest2 = new PutRequest(1, "clientId2", blobId2, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, null);
    DataInputStream putResponseStream = channel.sendAndReceive(putRequest2).getInputStream();
    PutResponse response2 = PutResponse.readFrom(putResponseStream);
    releaseNettyBufUnderneathStream(putResponseStream);
    assertEquals("Put blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, response2.getError());
    // get a blob properties on a stopped store, which should fail
    ArrayList<BlobId> ids = new ArrayList<>();
    ids.add(blobId1);
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<>();
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partitionId, ids);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest1 = new GetRequest(1, "clientId1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
    stream = channel.sendAndReceive(getRequest1).getInputStream();
    GetResponse resp1 = GetResponse.readFrom(stream, clusterMap);
    assertEquals("Get blob properties on stopped store should fail", ServerErrorCode.Replica_Unavailable, resp1.getPartitionResponseInfoList().get(0).getErrorCode());
    releaseNettyBufUnderneathStream(stream);
    // delete a blob on a stopped store, which should fail
    DeleteRequest deleteRequest = new DeleteRequest(1, "clientId1", blobId1, System.currentTimeMillis());
    stream = channel.sendAndReceive(deleteRequest).getInputStream();
    DeleteResponse deleteResponse = DeleteResponse.readFrom(stream);
    releaseNettyBufUnderneathStream(stream);
    assertEquals("Delete blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, deleteResponse.getError());
    // start the store via AdminRequest
    System.out.println("Begin to restart the BlobStore");
    adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionId, 1, "clientId");
    controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StartStore, adminRequest);
    stream = channel.sendAndReceive(controlRequest).getInputStream();
    adminResponse = AdminResponse.readFrom(stream);
    releaseNettyBufUnderneathStream(stream);
    assertEquals("Start store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
    List<? extends ReplicaId> replicaIds = partitionId.getReplicaIds();
    for (ReplicaId replicaId : replicaIds) {
        // forcibly mark replicas and disks as up.
        MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
        mockReplicaId.markReplicaDownStatus(false);
        ((MockDiskId) mockReplicaId.getDiskId()).setDiskState(HardwareState.AVAILABLE, false);
    }
    // put a blob on a restarted store , which should succeed
    putRequest2 = new PutRequest(1, "clientId2", blobId2, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, null);
    putResponseStream = channel.sendAndReceive(putRequest2).getInputStream();
    response2 = PutResponse.readFrom(putResponseStream);
    releaseNettyBufUnderneathStream(putResponseStream);
    assertEquals("Put blob on restarted store should succeed", ServerErrorCode.No_Error, response2.getError());
    // verify the put blob has been replicated successfully.
    notificationSystem.awaitBlobCreations(blobId2.getID());
    // get a blob on a restarted store , which should succeed
    ids = new ArrayList<BlobId>();
    ids.add(blobId2);
    partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    partitionRequestInfo = new PartitionRequestInfo(partitionId, ids);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest2 = new GetRequest(1, "clientId2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
    stream = channel.sendAndReceive(getRequest2).getInputStream();
    GetResponse resp2 = GetResponse.readFrom(stream, clusterMap);
    InputStream responseStream = resp2.getInputStream();
    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
    byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
    assertArrayEquals("Content mismatch.", data, actualBlobData);
    releaseNettyBufUnderneathStream(stream);
    // delete a blob on a restarted store , which should succeed
    deleteRequest = new DeleteRequest(1, "clientId2", blobId2, System.currentTimeMillis());
    stream = channel.sendAndReceive(deleteRequest).getInputStream();
    deleteResponse = DeleteResponse.readFrom(stream);
    releaseNettyBufUnderneathStream(stream);
    assertEquals("Delete blob on restarted store should succeed", ServerErrorCode.No_Error, deleteResponse.getError());
    router.close();
    connectionPool.shutdown();
}