Examples with BlockingChannel com.github.ambry.network.BlockingChannel used on opensource projects

Example 1 with BlockingChannel

use of com.github.ambry.network.BlockingChannel in project ambry by linkedin.

the class ServerHardDeleteTest method endToEndTestHardDeletes.

/**
 * Tests the hard delete functionality.
 * <p>
 * This test does the following:
 * 1. Makes 6 puts, waits for notification.
 * 2. Makes 2 deletes, waits for notification.
 * 3. Waits for hard deletes to catch up to the expected token value.
 * 4. Verifies that the two records that are deleted are zeroed out by hard deletes.
 * 5. Makes 3 more puts, waits for notification.
 * 6. Makes 3 deletes - 2 of records from the initial set of puts, and 1 from the new set.
 * 7. Waits for hard deletes to catch up again to the expected token value.
 * 8. Verifies that the three records that are deleted are zeroed out by hard deletes.
 *
 * @throws Exception
 */
@Test
public void endToEndTestHardDeletes() throws Exception {
    DataNodeId dataNodeId = mockClusterMap.getDataNodeIds().get(0);
    encryptionKey = new ArrayList<>(9);
    usermetadata = new ArrayList<>(9);
    data = new ArrayList<>(9);
    Random random = new Random();
    for (int i = 0; i < 9; i++) {
        if (i % 2 == 0) {
            encryptionKey.add(new byte[100]);
            random.nextBytes(encryptionKey.get(i));
        } else {
            encryptionKey.add(null);
        }
        usermetadata.add(new byte[1000 + i]);
        data.add(new byte[31870 + i]);
        random.nextBytes(usermetadata.get(i));
        random.nextBytes(data.get(i));
    }
    properties = new ArrayList<>(9);
    properties.add(new BlobProperties(31870, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31871, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false));
    properties.add(new BlobProperties(31872, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31873, "serviceid1", "ownerid", "jpeg", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false));
    properties.add(new BlobProperties(31874, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31875, "serviceid1", "ownerid", "jpeg", false, 0, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false));
    properties.add(new BlobProperties(31876, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    properties.add(new BlobProperties(31877, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false));
    properties.add(new BlobProperties(31878, "serviceid1", Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), true));
    List<PartitionId> partitionIds = mockClusterMap.getWritablePartitionIds();
    PartitionId chosenPartition = partitionIds.get(0);
    blobIdList = new ArrayList<>(9);
    for (int i = 0; i < 9; i++) {
        blobIdList.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, mockClusterMap.getLocalDatacenterId(), properties.get(i).getAccountId(), properties.get(i).getContainerId(), chosenPartition, false));
    }
    BlockingChannel channel = ServerTestUtil.getBlockingChannelBasedOnPortType(new Port(dataNodeId.getPort(), PortType.PLAINTEXT), "localhost", null, null);
    channel.connect();
    for (int i = 0; i < 6; i++) {
        // blob 3 and 5 are expired among these
        putBlob(blobIdList.get(i), properties.get(i), encryptionKey.get(i), usermetadata.get(i), data.get(i), channel);
    }
    notificationSystem.awaitBlobCreations(blobIdList.get(0).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(1).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(2).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(4).getID());
    // delete blob 1
    deleteBlob(blobIdList.get(1), channel);
    zeroOutBlobContent(1);
    // delete blob 4
    deleteBlob(blobIdList.get(4), channel);
    zeroOutBlobContent(4);
    notificationSystem.awaitBlobDeletions(blobIdList.get(1).getID());
    notificationSystem.awaitBlobDeletions(blobIdList.get(4).getID());
    time.sleep(TimeUnit.DAYS.toMillis(1));
    // Changes in this patch: a. New header version has 4 more bytes compared to previous b. BlobProperties increased by 1 byte
    // c. Encryption Key Record size is 114 for an encryptionKey of size 100. EncryptionKeyRecord could be null if not applicable.
    // Delta: 7 * 4 (headers for 7 records) + 1*6 (BlobProperties for 6 put records) + 114*3 = 376
    ensureCleanupTokenCatchesUp(chosenPartition.getReplicaIds().get(0).getReplicaPath(), mockClusterMap, 198896);
    getAndVerify(channel, 6);
    // put blob 6
    putBlob(blobIdList.get(6), properties.get(6), encryptionKey.get(6), usermetadata.get(6), data.get(6), channel);
    // put blob 7
    putBlob(blobIdList.get(7), properties.get(7), encryptionKey.get(7), usermetadata.get(7), data.get(7), channel);
    // put blob 8
    putBlob(blobIdList.get(8), properties.get(8), encryptionKey.get(8), usermetadata.get(8), data.get(8), channel);
    notificationSystem.awaitBlobCreations(blobIdList.get(6).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(7).getID());
    notificationSystem.awaitBlobCreations(blobIdList.get(8).getID());
    // Do more deletes
    // delete blob 3 that is expired.
    deleteBlob(blobIdList.get(3), channel);
    zeroOutBlobContent(3);
    // delete blob 0
    deleteBlob(blobIdList.get(0), channel);
    zeroOutBlobContent(0);
    // delete blob 6.
    deleteBlob(blobIdList.get(6), channel);
    zeroOutBlobContent(6);
    notificationSystem.awaitBlobDeletions(blobIdList.get(0).getID());
    notificationSystem.awaitBlobDeletions(blobIdList.get(6).getID());
    time.sleep(TimeUnit.DAYS.toMillis(1));
    // changes in this patch: a. New header version has 4 more bytes compared to preivous b. BlobProperties increased by 1 byte
    // c. Encryption Key Record size is 114 for an encryptionKey of size 100. EncryptionKeyRecord could be null if not applicable.
    // Delta: 6*4 (header) + 3*1 ( blob props) + 114*2 = 225 + 376 (from previous checkpoint) = 631
    ensureCleanupTokenCatchesUp(chosenPartition.getReplicaIds().get(0).getReplicaPath(), mockClusterMap, 298712);
    getAndVerify(channel, 9);
}

Example 2 with BlockingChannel

use of com.github.ambry.network.BlockingChannel in project ambry by linkedin.

the class ServerTestUtil method endToEndTest.

protected static void endToEndTest(Port targetPort, String routerDatacenter, String sslEnabledDatacenters, MockCluster cluster, SSLConfig clientSSLConfig, SSLSocketFactory clientSSLSocketFactory, Properties routerProps, boolean testEncryption) throws InterruptedException, IOException, InstantiationException {
    try {
        MockClusterMap clusterMap = cluster.getClusterMap();
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        byte[] usermetadata = new byte[1000];
        byte[] data = new byte[31870];
        byte[] encryptionKey = new byte[100];
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        BlobProperties properties = new BlobProperties(31870, "serviceid1", accountId, containerId, false);
        TestUtils.RANDOM.nextBytes(usermetadata);
        TestUtils.RANDOM.nextBytes(data);
        if (testEncryption) {
            TestUtils.RANDOM.nextBytes(encryptionKey);
        }
        List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds();
        short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
        BlobId blobId1 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false);
        BlobId blobId2 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false);
        BlobId blobId3 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false);
        BlobId blobId4 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false);
        // put blob 1
        PutRequest putRequest = new PutRequest(1, "client1", blobId1, properties, ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        BlockingChannel channel = getBlockingChannelBasedOnPortType(targetPort, "localhost", clientSSLSocketFactory, clientSSLConfig);
        channel.connect();
        channel.send(putRequest);
        InputStream putResponseStream = channel.receive().getInputStream();
        PutResponse response = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response.getError());
        // put blob 2
        PutRequest putRequest2 = new PutRequest(1, "client1", blobId2, properties, ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        channel.send(putRequest2);
        putResponseStream = channel.receive().getInputStream();
        PutResponse response2 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 3
        PutRequest putRequest3 = new PutRequest(1, "client1", blobId3, properties, ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        channel.send(putRequest3);
        putResponseStream = channel.receive().getInputStream();
        PutResponse response3 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // put blob 4 that is expired
        BlobProperties propertiesExpired = new BlobProperties(31870, "serviceid1", "ownerid", "jpeg", false, 0, accountId, containerId, false);
        PutRequest putRequest4 = new PutRequest(1, "client1", blobId4, propertiesExpired, ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        channel.send(putRequest4);
        putResponseStream = channel.receive().getInputStream();
        PutResponse response4 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response4.getError());
        // get blob properties
        ArrayList<BlobId> ids = new ArrayList<BlobId>();
        MockPartitionId partition = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        ids.add(blobId1);
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        channel.send(getRequest1);
        InputStream stream = channel.receive().getInputStream();
        GetResponse resp1 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // get blob properties with expired flag set
        ids = new ArrayList<BlobId>();
        partition = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.Include_Expired_Blobs);
        channel.send(getRequest1);
        stream = channel.receive().getInputStream();
        resp1 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // get blob properties for expired blob
        // 1. With no flag
        ArrayList<BlobId> idsExpired = new ArrayList<>();
        MockPartitionId partitionExpired = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        idsExpired.add(blobId4);
        ArrayList<PartitionRequestInfo> partitionRequestInfoListExpired = new ArrayList<>();
        PartitionRequestInfo partitionRequestInfoExpired = new PartitionRequestInfo(partitionExpired, idsExpired);
        partitionRequestInfoListExpired.add(partitionRequestInfoExpired);
        GetRequest getRequestExpired = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoListExpired, GetOption.None);
        channel.send(getRequestExpired);
        InputStream streamExpired = channel.receive().getInputStream();
        GetResponse respExpired = GetResponse.readFrom(new DataInputStream(streamExpired), clusterMap);
        assertEquals(ServerErrorCode.Blob_Expired, respExpired.getPartitionResponseInfoList().get(0).getErrorCode());
        // 2. With Include_Expired flag
        idsExpired = new ArrayList<>();
        partitionExpired = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        idsExpired.add(blobId4);
        partitionRequestInfoListExpired = new ArrayList<>();
        partitionRequestInfoExpired = new PartitionRequestInfo(partitionExpired, idsExpired);
        partitionRequestInfoListExpired.add(partitionRequestInfoExpired);
        getRequestExpired = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoListExpired, GetOption.Include_Expired_Blobs);
        channel.send(getRequestExpired);
        streamExpired = channel.receive().getInputStream();
        respExpired = GetResponse.readFrom(new DataInputStream(streamExpired), clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(respExpired.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("ownerid", propertyOutput.getOwnerId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // get user metadata
        GetRequest getRequest2 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        channel.send(getRequest2);
        stream = channel.receive().getInputStream();
        GetResponse resp2 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        try {
            ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp2.getInputStream());
            Assert.assertArrayEquals(usermetadata, userMetadataOutput.array());
            if (testEncryption) {
                assertNotNull("MessageMetadata should not have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
            } else {
                assertNull("MessageMetadata should have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            }
        } catch (MessageFormatException e) {
            assertEquals(false, true);
        }
        // get blob info
        GetRequest getRequest3 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobInfo, partitionRequestInfoList, GetOption.None);
        channel.send(getRequest3);
        stream = channel.receive().getInputStream();
        GetResponse resp3 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        InputStream responseStream = resp3.getInputStream();
        // verify blob properties.
        BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(responseStream);
        assertEquals(31870, propertyOutput.getBlobSize());
        assertEquals("serviceid1", propertyOutput.getServiceId());
        assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
        assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
        // verify user metadata
        ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(responseStream);
        Assert.assertArrayEquals(usermetadata, userMetadataOutput.array());
        if (testEncryption) {
            assertNotNull("MessageMetadata should not have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
        } else {
            assertNull("MessageMetadata should have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
        }
        // get blob all
        GetRequest getRequest4 = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        channel.send(getRequest4);
        stream = channel.receive().getInputStream();
        GetResponse resp4 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        responseStream = resp4.getInputStream();
        BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        byte[] actualBlobData = new byte[(int) blobAll.getBlobData().getSize()];
        blobAll.getBlobData().getStream().getByteBuffer().get(actualBlobData);
        // verify content
        Assert.assertArrayEquals("Content mismatch", data, actualBlobData);
        if (testEncryption) {
            Assert.assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
            Assert.assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
        } else {
            Assert.assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
        }
        // encryptionKey in this test doesn't have any relation to the content. Both are random bytes for test purposes.
        if (!testEncryption) {
            // Use router to get the blob
            Properties routerProperties = getRouterProps(routerDatacenter);
            routerProperties.putAll(routerProps);
            VerifiableProperties routerVerifiableProps = new VerifiableProperties(routerProperties);
            Router router = new NonBlockingRouterFactory(routerVerifiableProps, clusterMap, new MockNotificationSystem(9), getSSLFactoryIfRequired(routerVerifiableProps)).getRouter();
            checkBlobId(router, blobId1, data);
            router.close();
        }
        // fetch blob that does not exist
        // get blob properties
        ids = new ArrayList<BlobId>();
        partition = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        ids.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partition, false));
        partitionRequestInfoList.clear();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest5 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        channel.send(getRequest5);
        stream = channel.receive().getInputStream();
        GetResponse resp5 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        assertEquals(ServerErrorCode.Blob_Not_Found, resp5.getPartitionResponseInfoList().get(0).getErrorCode());
        channel.disconnect();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    }
}

Example 3 with BlockingChannel

use of com.github.ambry.network.BlockingChannel in project ambry by linkedin.

the class ServerTestUtil method endToEndReplicationWithMultiNodeSinglePartitionTest.

protected static void endToEndReplicationWithMultiNodeSinglePartitionTest(String routerDatacenter, String sslEnabledDatacenters, int interestedDataNodePortNumber, Port dataNode1Port, Port dataNode2Port, Port dataNode3Port, MockCluster cluster, SSLConfig clientSSLConfig1, SSLSocketFactory clientSSLSocketFactory1, MockNotificationSystem notificationSystem, Properties routerProps, boolean testEncryption) throws InterruptedException, IOException, InstantiationException {
    // interestedDataNodePortNumber is used to locate the datanode and hence has to be PlainText port
    try {
        int expectedTokenSize = 0;
        MockClusterMap clusterMap = cluster.getClusterMap();
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        ArrayList<BlobProperties> propertyList = new ArrayList<>();
        ArrayList<BlobId> blobIdList = new ArrayList<>();
        ArrayList<byte[]> dataList = new ArrayList<>();
        ArrayList<byte[]> encryptionKeyList = new ArrayList<>();
        byte[] usermetadata = new byte[1000];
        TestUtils.RANDOM.nextBytes(usermetadata);
        PartitionId partition = clusterMap.getWritablePartitionIds().get(0);
        for (int i = 0; i < 11; i++) {
            short accountId = Utils.getRandomShort(TestUtils.RANDOM);
            short containerId = Utils.getRandomShort(TestUtils.RANDOM);
            propertyList.add(new BlobProperties(1000, "serviceid1", accountId, containerId, testEncryption));
            blobIdList.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), accountId, containerId, partition, false));
            dataList.add(TestUtils.getRandomBytes(1000));
            if (testEncryption) {
                encryptionKeyList.add(TestUtils.getRandomBytes(128));
            } else {
                encryptionKeyList.add(null);
            }
        }
        // put blob 1
        PutRequest putRequest = new PutRequest(1, "client1", blobIdList.get(0), propertyList.get(0), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(0)), propertyList.get(0).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(0) != null ? ByteBuffer.wrap(encryptionKeyList.get(0)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(0), blobIdList.get(0), encryptionKeyList.get(0) != null ? ByteBuffer.wrap(encryptionKeyList.get(0)) : null, ByteBuffer.wrap(usermetadata), dataList.get(0));
        BlockingChannel channel1 = getBlockingChannelBasedOnPortType(dataNode1Port, "localhost", clientSSLSocketFactory1, clientSSLConfig1);
        BlockingChannel channel2 = getBlockingChannelBasedOnPortType(dataNode2Port, "localhost", clientSSLSocketFactory1, clientSSLConfig1);
        BlockingChannel channel3 = getBlockingChannelBasedOnPortType(dataNode3Port, "localhost", clientSSLSocketFactory1, clientSSLConfig1);
        channel1.connect();
        channel2.connect();
        channel3.connect();
        channel1.send(putRequest);
        InputStream putResponseStream = channel1.receive().getInputStream();
        PutResponse response = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response.getError());
        // put blob 2
        PutRequest putRequest2 = new PutRequest(1, "client1", blobIdList.get(1), propertyList.get(1), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(1)), propertyList.get(1).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(1) != null ? ByteBuffer.wrap(encryptionKeyList.get(1)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(1), blobIdList.get(1), encryptionKeyList.get(1) != null ? ByteBuffer.wrap(encryptionKeyList.get(1)) : null, ByteBuffer.wrap(usermetadata), dataList.get(1));
        channel2.send(putRequest2);
        putResponseStream = channel2.receive().getInputStream();
        PutResponse response2 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 3
        PutRequest putRequest3 = new PutRequest(1, "client1", blobIdList.get(2), propertyList.get(2), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(2)), propertyList.get(2).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(2) != null ? ByteBuffer.wrap(encryptionKeyList.get(2)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(2), blobIdList.get(2), encryptionKeyList.get(2) != null ? ByteBuffer.wrap(encryptionKeyList.get(2)) : null, ByteBuffer.wrap(usermetadata), dataList.get(2));
        channel3.send(putRequest3);
        putResponseStream = channel3.receive().getInputStream();
        PutResponse response3 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // put blob 4
        putRequest = new PutRequest(1, "client1", blobIdList.get(3), propertyList.get(3), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(3)), propertyList.get(3).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(3) != null ? ByteBuffer.wrap(encryptionKeyList.get(3)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(3), blobIdList.get(3), encryptionKeyList.get(3) != null ? ByteBuffer.wrap(encryptionKeyList.get(3)) : null, ByteBuffer.wrap(usermetadata), dataList.get(3));
        channel1.send(putRequest);
        putResponseStream = channel1.receive().getInputStream();
        response = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response.getError());
        // put blob 5
        putRequest2 = new PutRequest(1, "client1", blobIdList.get(4), propertyList.get(4), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(4)), propertyList.get(4).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(4) != null ? ByteBuffer.wrap(encryptionKeyList.get(4)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(4), blobIdList.get(4), encryptionKeyList.get(4) != null ? ByteBuffer.wrap(encryptionKeyList.get(4)) : null, ByteBuffer.wrap(usermetadata), dataList.get(4));
        channel2.send(putRequest2);
        putResponseStream = channel2.receive().getInputStream();
        response2 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 6
        putRequest3 = new PutRequest(1, "client1", blobIdList.get(5), propertyList.get(5), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(5)), propertyList.get(5).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(5) != null ? ByteBuffer.wrap(encryptionKeyList.get(5)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(5), blobIdList.get(5), encryptionKeyList.get(5) != null ? ByteBuffer.wrap(encryptionKeyList.get(5)) : null, ByteBuffer.wrap(usermetadata), dataList.get(5));
        channel3.send(putRequest3);
        putResponseStream = channel3.receive().getInputStream();
        response3 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // wait till replication can complete
        notificationSystem.awaitBlobCreations(blobIdList.get(0).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(1).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(2).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(3).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(4).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(5).getID());
        // get blob properties
        ArrayList<BlobId> ids = new ArrayList<BlobId>();
        MockPartitionId mockPartitionId = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        ids.add(blobIdList.get(2));
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(mockPartitionId, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        channel2.send(getRequest1);
        InputStream stream = channel2.receive().getInputStream();
        GetResponse resp1 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        assertEquals(ServerErrorCode.No_Error, resp1.getError());
        assertEquals(ServerErrorCode.No_Error, resp1.getPartitionResponseInfoList().get(0).getErrorCode());
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(1000, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", propertyList.get(2).getAccountId(), propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", propertyList.get(2).getContainerId(), propertyOutput.getContainerId());
            assertEquals("IsEncrypted mismatch", propertyList.get(2).isEncrypted(), propertyOutput.isEncrypted());
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // get user metadata
        ids.clear();
        ids.add(blobIdList.get(1));
        GetRequest getRequest2 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        channel1.send(getRequest2);
        stream = channel1.receive().getInputStream();
        GetResponse resp2 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        assertEquals(ServerErrorCode.No_Error, resp2.getError());
        assertEquals(ServerErrorCode.No_Error, resp2.getPartitionResponseInfoList().get(0).getErrorCode());
        try {
            ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp2.getInputStream());
            Assert.assertArrayEquals(usermetadata, userMetadataOutput.array());
            if (testEncryption) {
                assertNotNull("MessageMetadata should not have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                assertArrayEquals("EncryptionKey mismatch", encryptionKeyList.get(1), resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
            } else {
                assertNull("MessageMetadata should have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            }
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // get blob
        ids.clear();
        ids.add(blobIdList.get(0));
        GetRequest getRequest3 = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
        channel3.send(getRequest3);
        stream = channel3.receive().getInputStream();
        GetResponse resp3 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        try {
            BlobData blobData = MessageFormatRecord.deserializeBlob(resp3.getInputStream());
            byte[] blobout = new byte[(int) blobData.getSize()];
            int readsize = 0;
            while (readsize < blobData.getSize()) {
                readsize += blobData.getStream().read(blobout, readsize, (int) blobData.getSize() - readsize);
            }
            Assert.assertArrayEquals(dataList.get(0), blobout);
            if (testEncryption) {
                assertNotNull("MessageMetadata should not have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                assertArrayEquals("EncryptionKey mismatch", encryptionKeyList.get(0), resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
            } else {
                assertNull("MessageMetadata should have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            }
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // get blob all
        ids.clear();
        ids.add(blobIdList.get(0));
        GetRequest getRequest4 = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        channel1.send(getRequest4);
        stream = channel1.receive().getInputStream();
        GetResponse resp4 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        try {
            BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp4.getInputStream(), blobIdFactory);
            byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
            int readsize = 0;
            while (readsize < blobAll.getBlobData().getSize()) {
                readsize += blobAll.getBlobData().getStream().read(blobout, readsize, (int) blobAll.getBlobData().getSize() - readsize);
            }
            Assert.assertArrayEquals(dataList.get(0), blobout);
            if (testEncryption) {
                assertNotNull("MessageMetadata should not have been null", blobAll.getBlobEncryptionKey());
                assertArrayEquals("EncryptionKey mismatch", encryptionKeyList.get(0), blobAll.getBlobEncryptionKey().array());
            } else {
                assertNull("MessageMetadata should have been null", blobAll.getBlobEncryptionKey());
            }
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        if (!testEncryption) {
            // get blob data
            // Use router to get the blob
            Properties routerProperties = getRouterProps(routerDatacenter);
            routerProperties.putAll(routerProps);
            VerifiableProperties routerVerifiableProperties = new VerifiableProperties(routerProperties);
            Router router = new NonBlockingRouterFactory(routerVerifiableProperties, clusterMap, notificationSystem, getSSLFactoryIfRequired(routerVerifiableProperties)).getRouter();
            checkBlobId(router, blobIdList.get(0), dataList.get(0));
            checkBlobId(router, blobIdList.get(1), dataList.get(1));
            checkBlobId(router, blobIdList.get(2), dataList.get(2));
            checkBlobId(router, blobIdList.get(3), dataList.get(3));
            checkBlobId(router, blobIdList.get(4), dataList.get(4));
            checkBlobId(router, blobIdList.get(5), dataList.get(5));
            router.close();
        }
        // fetch blob that does not exist
        // get blob properties
        ids = new ArrayList<BlobId>();
        mockPartitionId = (MockPartitionId) clusterMap.getWritablePartitionIds().get(0);
        ids.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), propertyList.get(0).getAccountId(), propertyList.get(0).getContainerId(), mockPartitionId, false));
        partitionRequestInfoList.clear();
        partitionRequestInfo = new PartitionRequestInfo(mockPartitionId, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest5 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        channel3.send(getRequest5);
        stream = channel3.receive().getInputStream();
        GetResponse resp5 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        assertEquals(ServerErrorCode.No_Error, resp5.getError());
        assertEquals(ServerErrorCode.Blob_Not_Found, resp5.getPartitionResponseInfoList().get(0).getErrorCode());
        // delete a blob and ensure it is propagated
        DeleteRequest deleteRequest = new DeleteRequest(1, "reptest", blobIdList.get(0), System.currentTimeMillis());
        expectedTokenSize += getDeleteRecordSize(blobIdList.get(0));
        channel1.send(deleteRequest);
        InputStream deleteResponseStream = channel1.receive().getInputStream();
        DeleteResponse deleteResponse = DeleteResponse.readFrom(new DataInputStream(deleteResponseStream));
        assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        notificationSystem.awaitBlobDeletions(blobIdList.get(0).getID());
        ids = new ArrayList<BlobId>();
        ids.add(blobIdList.get(0));
        partitionRequestInfoList.clear();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest6 = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
        channel3.send(getRequest6);
        stream = channel3.receive().getInputStream();
        GetResponse resp6 = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
        assertEquals(ServerErrorCode.No_Error, resp6.getError());
        assertEquals(ServerErrorCode.Blob_Deleted, resp6.getPartitionResponseInfoList().get(0).getErrorCode());
        // get the data node to inspect replication tokens on
        DataNodeId dataNodeId = clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
        // read the replica file and check correctness
        // The token offset value of 13098 was derived as followed:
        // - Up to this point we have done 6 puts and 1 delete
        // - Each put takes up 2183 bytes in the log (1000 data, 1000 user metadata, 183 ambry metadata)
        // - Each delete takes up 97 bytes in the log
        // - The offset stored in the token will be the position of the last entry in the log (the delete, in this case)
        // - Thus, it will be at the end of the 6 puts: 6 * 2183 = 13098
        checkReplicaTokens(clusterMap, dataNodeId, expectedTokenSize - getDeleteRecordSize(blobIdList.get(0)), "0");
        // Shut down server 1
        cluster.getServers().get(0).shutdown();
        cluster.getServers().get(0).awaitShutdown();
        // Add more data to server 2 and server 3. Recover server 1 and ensure it is completely replicated
        // put blob 7
        putRequest2 = new PutRequest(1, "client1", blobIdList.get(6), propertyList.get(6), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(6)), propertyList.get(6).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(6) != null ? ByteBuffer.wrap(encryptionKeyList.get(6)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(6), blobIdList.get(6), encryptionKeyList.get(6) != null ? ByteBuffer.wrap(encryptionKeyList.get(6)) : null, ByteBuffer.wrap(usermetadata), dataList.get(6));
        channel2.send(putRequest2);
        putResponseStream = channel2.receive().getInputStream();
        response2 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 8
        putRequest3 = new PutRequest(1, "client1", blobIdList.get(7), propertyList.get(7), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(7)), propertyList.get(7).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(7) != null ? ByteBuffer.wrap(encryptionKeyList.get(7)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(7), blobIdList.get(7), encryptionKeyList.get(7) != null ? ByteBuffer.wrap(encryptionKeyList.get(7)) : null, ByteBuffer.wrap(usermetadata), dataList.get(7));
        channel3.send(putRequest3);
        putResponseStream = channel3.receive().getInputStream();
        response3 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // put blob 9
        putRequest2 = new PutRequest(1, "client1", blobIdList.get(8), propertyList.get(8), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(8)), propertyList.get(8).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(8) != null ? ByteBuffer.wrap(encryptionKeyList.get(8)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(8), blobIdList.get(8), encryptionKeyList.get(8) != null ? ByteBuffer.wrap(encryptionKeyList.get(8)) : null, ByteBuffer.wrap(usermetadata), dataList.get(8));
        channel2.send(putRequest2);
        putResponseStream = channel2.receive().getInputStream();
        response2 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 10
        putRequest3 = new PutRequest(1, "client1", blobIdList.get(9), propertyList.get(9), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(9)), propertyList.get(9).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(9) != null ? ByteBuffer.wrap(encryptionKeyList.get(9)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(9), blobIdList.get(9), encryptionKeyList.get(9) != null ? ByteBuffer.wrap(encryptionKeyList.get(9)) : null, ByteBuffer.wrap(usermetadata), dataList.get(9));
        channel3.send(putRequest3);
        putResponseStream = channel3.receive().getInputStream();
        response3 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // put blob 11
        putRequest2 = new PutRequest(1, "client1", blobIdList.get(10), propertyList.get(10), ByteBuffer.wrap(usermetadata), ByteBuffer.wrap(dataList.get(10)), propertyList.get(10).getBlobSize(), BlobType.DataBlob, encryptionKeyList.get(10) != null ? ByteBuffer.wrap(encryptionKeyList.get(10)) : null);
        expectedTokenSize += getPutRecordSize(propertyList.get(10), blobIdList.get(10), encryptionKeyList.get(10) != null ? ByteBuffer.wrap(encryptionKeyList.get(10)) : null, ByteBuffer.wrap(usermetadata), dataList.get(10));
        channel2.send(putRequest2);
        putResponseStream = channel2.receive().getInputStream();
        response2 = PutResponse.readFrom(new DataInputStream(putResponseStream));
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        cluster.getServers().get(0).startup();
        // wait for server to recover
        notificationSystem.awaitBlobCreations(blobIdList.get(6).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(7).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(8).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(9).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(10).getID());
        channel1.disconnect();
        channel1.connect();
        // get blob
        try {
            checkBlobContent(clusterMap, blobIdList.get(1), channel1, dataList.get(1), encryptionKeyList.get(1));
            checkBlobContent(clusterMap, blobIdList.get(2), channel1, dataList.get(2), encryptionKeyList.get(2));
            checkBlobContent(clusterMap, blobIdList.get(3), channel1, dataList.get(3), encryptionKeyList.get(3));
            checkBlobContent(clusterMap, blobIdList.get(4), channel1, dataList.get(4), encryptionKeyList.get(4));
            checkBlobContent(clusterMap, blobIdList.get(5), channel1, dataList.get(5), encryptionKeyList.get(5));
            checkBlobContent(clusterMap, blobIdList.get(6), channel1, dataList.get(6), encryptionKeyList.get(6));
            checkBlobContent(clusterMap, blobIdList.get(7), channel1, dataList.get(7), encryptionKeyList.get(7));
            checkBlobContent(clusterMap, blobIdList.get(8), channel1, dataList.get(8), encryptionKeyList.get(8));
            checkBlobContent(clusterMap, blobIdList.get(9), channel1, dataList.get(9), encryptionKeyList.get(9));
            checkBlobContent(clusterMap, blobIdList.get(10), channel1, dataList.get(10), encryptionKeyList.get(10));
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        // Shutdown server 1. Remove all its data from all mount path. Recover server 1 and ensure node is built
        cluster.getServers().get(0).shutdown();
        cluster.getServers().get(0).awaitShutdown();
        File mountFile = new File(clusterMap.getReplicaIds(dataNodeId).get(0).getMountPath());
        for (File toDelete : mountFile.listFiles()) {
            deleteFolderContent(toDelete, true);
        }
        notificationSystem.decrementCreatedReplica(blobIdList.get(1).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(2).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(3).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(4).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(5).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(6).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(7).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(8).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(9).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        notificationSystem.decrementCreatedReplica(blobIdList.get(10).getID(), dataNodeId.getHostname(), dataNodeId.getPort());
        cluster.getServers().get(0).startup();
        notificationSystem.awaitBlobCreations(blobIdList.get(1).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(2).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(3).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(4).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(5).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(6).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(7).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(8).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(9).getID());
        notificationSystem.awaitBlobCreations(blobIdList.get(10).getID());
        channel1.disconnect();
        channel1.connect();
        // get blob
        try {
            checkBlobContent(clusterMap, blobIdList.get(1), channel1, dataList.get(1), encryptionKeyList.get(1));
            checkBlobContent(clusterMap, blobIdList.get(2), channel1, dataList.get(2), encryptionKeyList.get(2));
            checkBlobContent(clusterMap, blobIdList.get(3), channel1, dataList.get(3), encryptionKeyList.get(3));
            checkBlobContent(clusterMap, blobIdList.get(4), channel1, dataList.get(4), encryptionKeyList.get(4));
            checkBlobContent(clusterMap, blobIdList.get(5), channel1, dataList.get(5), encryptionKeyList.get(5));
            checkBlobContent(clusterMap, blobIdList.get(6), channel1, dataList.get(6), encryptionKeyList.get(6));
            checkBlobContent(clusterMap, blobIdList.get(7), channel1, dataList.get(7), encryptionKeyList.get(7));
            checkBlobContent(clusterMap, blobIdList.get(8), channel1, dataList.get(8), encryptionKeyList.get(8));
            checkBlobContent(clusterMap, blobIdList.get(9), channel1, dataList.get(9), encryptionKeyList.get(9));
            checkBlobContent(clusterMap, blobIdList.get(10), channel1, dataList.get(10), encryptionKeyList.get(10));
        } catch (MessageFormatException e) {
            Assert.fail();
        }
        channel1.disconnect();
        channel2.disconnect();
        channel3.disconnect();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    }
}

Example 4 with BlockingChannel

use of com.github.ambry.network.BlockingChannel in project ambry by linkedin.

the class ServerTestUtil method endToEndReplicationWithMultiNodeMultiPartitionTest.

protected static void endToEndReplicationWithMultiNodeMultiPartitionTest(int interestedDataNodePortNumber, Port dataNode1Port, Port dataNode2Port, Port dataNode3Port, MockCluster cluster, SSLConfig clientSSLConfig1, SSLConfig clientSSLConfig2, SSLConfig clientSSLConfig3, SSLSocketFactory clientSSLSocketFactory1, SSLSocketFactory clientSSLSocketFactory2, SSLSocketFactory clientSSLSocketFactory3, MockNotificationSystem notificationSystem, boolean testEncryption) throws InterruptedException, IOException, InstantiationException {
    // interestedDataNodePortNumber is used to locate the datanode and hence has to be PlainTextPort
    try {
        MockClusterMap clusterMap = cluster.getClusterMap();
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        List<AmbryServer> serverList = cluster.getServers();
        byte[] usermetadata = new byte[100];
        byte[] data = new byte[100];
        byte[] encryptionKey = null;
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        BlobProperties properties = new BlobProperties(100, "serviceid1", accountId, containerId, false);
        TestUtils.RANDOM.nextBytes(usermetadata);
        TestUtils.RANDOM.nextBytes(data);
        if (testEncryption) {
            encryptionKey = new byte[100];
            TestUtils.RANDOM.nextBytes(encryptionKey);
        }
        // connect to all the servers
        BlockingChannel channel1 = getBlockingChannelBasedOnPortType(dataNode1Port, "localhost", clientSSLSocketFactory1, clientSSLConfig1);
        BlockingChannel channel2 = getBlockingChannelBasedOnPortType(dataNode2Port, "localhost", clientSSLSocketFactory2, clientSSLConfig2);
        BlockingChannel channel3 = getBlockingChannelBasedOnPortType(dataNode3Port, "localhost", clientSSLSocketFactory3, clientSSLConfig3);
        // put all the blobs to random servers
        channel1.connect();
        channel2.connect();
        channel3.connect();
        int noOfParallelThreads = 3;
        CountDownLatch latch = new CountDownLatch(noOfParallelThreads);
        List<DirectSender> runnables = new ArrayList<DirectSender>(noOfParallelThreads);
        BlockingChannel channel = null;
        for (int i = 0; i < noOfParallelThreads; i++) {
            if (i % noOfParallelThreads == 0) {
                channel = channel1;
            } else if (i % noOfParallelThreads == 1) {
                channel = channel2;
            } else if (i % noOfParallelThreads == 2) {
                channel = channel3;
            }
            DirectSender runnable = new DirectSender(cluster, channel, 50, data, usermetadata, properties, encryptionKey, latch);
            runnables.add(runnable);
            Thread threadToRun = new Thread(runnable);
            threadToRun.start();
        }
        assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
        // wait till replication can complete
        List<BlobId> blobIds = new ArrayList<BlobId>();
        for (int i = 0; i < runnables.size(); i++) {
            blobIds.addAll(runnables.get(i).getBlobIds());
        }
        for (BlobId blobId : blobIds) {
            notificationSystem.awaitBlobCreations(blobId.getID());
        }
        // Now that the blob is created and replicated, test the cases where a put request arrives for the same blob id
        // later than replication.
        testLatePutRequest(blobIds.get(0), properties, usermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.No_Error);
        // Test the case where a put arrives with the same id as one in the server, but the blob is not identical.
        BlobProperties differentProperties = new BlobProperties(properties.getBlobSize(), properties.getServiceId(), accountId, containerId, testEncryption);
        testLatePutRequest(blobIds.get(0), differentProperties, usermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
        byte[] differentUsermetadata = Arrays.copyOf(usermetadata, usermetadata.length);
        differentUsermetadata[0] = (byte) ~differentUsermetadata[0];
        testLatePutRequest(blobIds.get(0), properties, differentUsermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
        byte[] differentData = Arrays.copyOf(data, data.length);
        differentData[0] = (byte) ~differentData[0];
        testLatePutRequest(blobIds.get(0), properties, usermetadata, differentData, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
        // verify blob properties, metadata and blob across all nodes
        for (int i = 0; i < 3; i++) {
            channel = null;
            if (i == 0) {
                channel = channel1;
            } else if (i == 1) {
                channel = channel2;
            } else if (i == 2) {
                channel = channel3;
            }
            ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
            for (int j = 0; j < blobIds.size(); j++) {
                ArrayList<BlobId> ids = new ArrayList<BlobId>();
                ids.add(blobIds.get(j));
                partitionRequestInfoList.clear();
                PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
                partitionRequestInfoList.add(partitionRequestInfo);
                GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
                channel.send(getRequest);
                InputStream stream = channel.receive().getInputStream();
                GetResponse resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
                try {
                    BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                    assertEquals(100, propertyOutput.getBlobSize());
                    assertEquals("serviceid1", propertyOutput.getServiceId());
                    assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                    assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
                // get user metadata
                getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
                channel.send(getRequest);
                stream = channel.receive().getInputStream();
                resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
                try {
                    ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                    Assert.assertArrayEquals(usermetadata, userMetadataOutput.array());
                    if (testEncryption) {
                        assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                    } else {
                        assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    }
                } catch (MessageFormatException e) {
                    e.printStackTrace();
                    Assert.fail();
                }
                // get blob
                getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
                channel.send(getRequest);
                stream = channel.receive().getInputStream();
                resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
                try {
                    BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                    byte[] blobout = new byte[(int) blobData.getSize()];
                    int readsize = 0;
                    while (readsize < blobData.getSize()) {
                        readsize += blobData.getStream().read(blobout, readsize, (int) blobData.getSize() - readsize);
                    }
                    Assert.assertArrayEquals(data, blobout);
                    if (testEncryption) {
                        assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                    } else {
                        assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    }
                } catch (MessageFormatException e) {
                    e.printStackTrace();
                    Assert.fail();
                }
                // get blob all
                getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
                channel.send(getRequest);
                stream = channel.receive().getInputStream();
                resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
                try {
                    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                    byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
                    int readsize = 0;
                    while (readsize < blobAll.getBlobData().getSize()) {
                        readsize += blobAll.getBlobData().getStream().read(blobout, readsize, (int) blobAll.getBlobData().getSize() - readsize);
                    }
                    Assert.assertArrayEquals(data, blobout);
                    if (testEncryption) {
                        Assert.assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                        Assert.assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                    } else {
                        Assert.assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                    }
                } catch (MessageFormatException e) {
                    e.printStackTrace();
                    Assert.fail();
                }
            }
        }
        // delete random blobs, wait for replication and ensure it is deleted in all nodes
        Set<BlobId> blobsDeleted = new HashSet<BlobId>();
        Set<BlobId> blobsChecked = new HashSet<BlobId>();
        for (int i = 0; i < blobIds.size(); i++) {
            int j = new Random().nextInt(3);
            if (j == 0) {
                j = new Random().nextInt(3);
                if (j == 0) {
                    channel = channel1;
                } else if (j == 1) {
                    channel = channel2;
                } else if (j == 2) {
                    channel = channel3;
                }
                DeleteRequest deleteRequest = new DeleteRequest(1, "reptest", blobIds.get(i), System.currentTimeMillis());
                channel.send(deleteRequest);
                InputStream deleteResponseStream = channel.receive().getInputStream();
                DeleteResponse deleteResponse = DeleteResponse.readFrom(new DataInputStream(deleteResponseStream));
                assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
                blobsDeleted.add(blobIds.get(i));
            }
        }
        Iterator<BlobId> iterator = blobsDeleted.iterator();
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        while (iterator.hasNext()) {
            BlobId deletedId = iterator.next();
            notificationSystem.awaitBlobDeletions(deletedId.getID());
            for (int j = 0; j < 3; j++) {
                if (j == 0) {
                    channel = channel1;
                } else if (j == 1) {
                    channel = channel2;
                } else if (j == 2) {
                    channel = channel3;
                }
                ArrayList<BlobId> ids = new ArrayList<BlobId>();
                ids.add(deletedId);
                partitionRequestInfoList.clear();
                PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(deletedId.getPartition(), ids);
                partitionRequestInfoList.add(partitionRequestInfo);
                GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
                channel.send(getRequest);
                InputStream stream = channel.receive().getInputStream();
                GetResponse resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
                assertEquals(ServerErrorCode.Blob_Deleted, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            }
        }
        // take a server down, clean up a mount path, start and ensure replication fixes it
        serverList.get(0).shutdown();
        serverList.get(0).awaitShutdown();
        MockDataNodeId dataNode = (MockDataNodeId) clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
        System.out.println("Cleaning mount path " + dataNode.getMountPaths().get(0));
        for (ReplicaId replicaId : clusterMap.getReplicaIds(dataNode)) {
            if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(0)) == 0) {
                System.out.println("Cleaning partition " + replicaId.getPartitionId());
            }
        }
        deleteFolderContent(new File(dataNode.getMountPaths().get(0)), false);
        for (int i = 0; i < blobIds.size(); i++) {
            for (ReplicaId replicaId : blobIds.get(i).getPartition().getReplicaIds()) {
                if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(0)) == 0) {
                    if (blobsDeleted.contains(blobIds.get(i))) {
                        notificationSystem.decrementDeletedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
                    } else {
                        notificationSystem.decrementCreatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
                    }
                }
            }
        }
        serverList.get(0).startup();
        channel1.disconnect();
        channel1.connect();
        for (int j = 0; j < blobIds.size(); j++) {
            if (blobsDeleted.contains(blobIds.get(j))) {
                notificationSystem.awaitBlobDeletions(blobIds.get(j).getID());
            } else {
                notificationSystem.awaitBlobCreations(blobIds.get(j).getID());
            }
            ArrayList<BlobId> ids = new ArrayList<BlobId>();
            ids.add(blobIds.get(j));
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            InputStream stream = channel1.receive().getInputStream();
            GetResponse resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsDeleted.contains(blobIds.get(j)));
            } else {
                try {
                    BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                    assertEquals(100, propertyOutput.getBlobSize());
                    assertEquals("serviceid1", propertyOutput.getServiceId());
                    assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                    assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
            // get user metadata
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            stream = channel1.receive().getInputStream();
            resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsDeleted.contains(blobIds.get(j)));
            } else {
                try {
                    ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                    Assert.assertArrayEquals(usermetadata, userMetadataOutput.array());
                    if (testEncryption) {
                        assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                    } else {
                        assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    }
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
            // get blob
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            stream = channel1.receive().getInputStream();
            resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsDeleted.contains(blobIds.get(j)));
            } else {
                try {
                    BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                    byte[] blobout = new byte[(int) blobData.getSize()];
                    int readsize = 0;
                    while (readsize < blobData.getSize()) {
                        readsize += blobData.getStream().read(blobout, readsize, (int) blobData.getSize() - readsize);
                    }
                    Assert.assertArrayEquals(data, blobout);
                    if (testEncryption) {
                        assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                    } else {
                        assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    }
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
            // get blob all
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            stream = channel1.receive().getInputStream();
            resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsDeleted.contains(blobIds.get(j)));
                blobsDeleted.remove(blobIds.get(j));
                blobsChecked.add(blobIds.get(j));
            } else {
                try {
                    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                    byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
                    int readsize = 0;
                    while (readsize < blobAll.getBlobData().getSize()) {
                        readsize += blobAll.getBlobData().getStream().read(blobout, readsize, (int) blobAll.getBlobData().getSize() - readsize);
                    }
                    Assert.assertArrayEquals(data, blobout);
                    if (testEncryption) {
                        Assert.assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                        Assert.assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                    } else {
                        Assert.assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                    }
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
        }
        assertEquals(0, blobsDeleted.size());
        // take a server down, clean all contents, start and ensure replication fixes it
        serverList.get(0).shutdown();
        serverList.get(0).awaitShutdown();
        dataNode = (MockDataNodeId) clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
        for (int i = 0; i < dataNode.getMountPaths().size(); i++) {
            System.out.println("Cleaning mount path " + dataNode.getMountPaths().get(i));
            for (ReplicaId replicaId : clusterMap.getReplicaIds(dataNode)) {
                if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(i)) == 0) {
                    System.out.println("Cleaning partition " + replicaId.getPartitionId());
                }
            }
            deleteFolderContent(new File(dataNode.getMountPaths().get(i)), false);
        }
        for (int i = 0; i < blobIds.size(); i++) {
            if (blobsChecked.contains(blobIds.get(i))) {
                notificationSystem.decrementDeletedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
            } else {
                notificationSystem.decrementCreatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
            }
        }
        serverList.get(0).startup();
        channel1.disconnect();
        channel1.connect();
        for (int j = 0; j < blobIds.size(); j++) {
            if (blobsChecked.contains(blobIds.get(j))) {
                notificationSystem.awaitBlobDeletions(blobIds.get(j).getID());
            } else {
                notificationSystem.awaitBlobCreations(blobIds.get(j).getID());
            }
            ArrayList<BlobId> ids = new ArrayList<BlobId>();
            ids.add(blobIds.get(j));
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            InputStream stream = channel1.receive().getInputStream();
            GetResponse resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsChecked.contains(blobIds.get(j)));
            } else {
                try {
                    BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                    assertEquals(100, propertyOutput.getBlobSize());
                    assertEquals("serviceid1", propertyOutput.getServiceId());
                    assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                    assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
            // get user metadata
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            stream = channel1.receive().getInputStream();
            resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsChecked.contains(blobIds.get(j)));
            } else {
                try {
                    ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                    Assert.assertArrayEquals(usermetadata, userMetadataOutput.array());
                    if (testEncryption) {
                        assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                    } else {
                        assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    }
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
            // get blob
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            stream = channel1.receive().getInputStream();
            resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsChecked.contains(blobIds.get(j)));
            } else {
                try {
                    BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                    byte[] blobout = new byte[(int) blobData.getSize()];
                    int readsize = 0;
                    while (readsize < blobData.getSize()) {
                        readsize += blobData.getStream().read(blobout, readsize, (int) blobData.getSize() - readsize);
                    }
                    Assert.assertArrayEquals(data, blobout);
                    if (testEncryption) {
                        assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                        assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                    } else {
                        assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    }
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
            // get blob all
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
            channel1.send(getRequest);
            stream = channel1.receive().getInputStream();
            resp = GetResponse.readFrom(new DataInputStream(stream), clusterMap);
            if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
                Assert.assertTrue(blobsChecked.contains(blobIds.get(j)));
                blobsChecked.remove(blobIds.get(j));
            } else {
                try {
                    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                    byte[] blobout = new byte[(int) blobAll.getBlobData().getSize()];
                    int readsize = 0;
                    while (readsize < blobAll.getBlobData().getSize()) {
                        readsize += blobAll.getBlobData().getStream().read(blobout, readsize, (int) blobAll.getBlobData().getSize() - readsize);
                    }
                    Assert.assertArrayEquals(data, blobout);
                    if (testEncryption) {
                        Assert.assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                        Assert.assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                    } else {
                        Assert.assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                    }
                } catch (MessageFormatException e) {
                    Assert.fail();
                }
            }
        }
        assertEquals(0, blobsChecked.size());
        channel1.disconnect();
        channel2.disconnect();
        channel3.disconnect();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    }
}