Examples with UndeleteRequest com.github.ambry.protocol.UndeleteRequest used on opensource projects

Example 1 with UndeleteRequest

use of com.github.ambry.protocol.UndeleteRequest in project ambry by linkedin.

the class UndeleteOperation method fetchRequests.

/**
 * Fetch {@link UndeleteRequest}s to send for the operation.
 * @param requestRegistrationCallback the {@link RequestRegistrationCallback} to use for addition of requests that
 *                                    need to be sent to the storage server
 */
private void fetchRequests(RequestRegistrationCallback<UndeleteOperation> requestRegistrationCallback) {
    Iterator<ReplicaId> replicaIterator = operationTracker.getReplicaIterator();
    while (replicaIterator.hasNext()) {
        ReplicaId replica = replicaIterator.next();
        String hostname = replica.getDataNodeId().getHostname();
        Port port = RouterUtils.getPortToConnectTo(replica, routerConfig.routerEnableHttp2NetworkClient);
        UndeleteRequest undeleteRequest = createUndeleteRequest();
        undeleteRequestInfos.put(undeleteRequest.getCorrelationId(), new UndeleteRequestInfo(time.milliseconds(), replica));
        RequestInfo requestInfo = new RequestInfo(hostname, port, undeleteRequest, replica, operationQuotaCharger);
        requestRegistrationCallback.registerRequestToSend(this, requestInfo);
        replicaIterator.remove();
        if (RouterUtils.isRemoteReplica(routerConfig, replica)) {
            LOGGER.trace("Making request with correlationId {} to a remote replica {} in {} ", undeleteRequest.getCorrelationId(), replica.getDataNodeId(), replica.getDataNodeId().getDatacenterName());
            routerMetrics.crossColoRequestCount.inc();
        } else {
            LOGGER.trace("Making request with correlationId {} to a local replica {} ", undeleteRequest.getCorrelationId(), replica.getDataNodeId());
        }
        routerMetrics.getDataNodeBasedMetrics(replica.getDataNodeId()).undeleteRequestRate.mark();
    }
}

Example 2 with UndeleteRequest

use of com.github.ambry.protocol.UndeleteRequest in project ambry by linkedin.

the class StoredBlob method send.

/**
 * Take in a request in the form of {@link Send} and return a response in the form of a
 * {@link BoundedNettyByteBufReceive}.
 * @param send the request.
 * @return the response.
 * @throws IOException if there was an error in interpreting the request.
 */
public BoundedNettyByteBufReceive send(Send send) throws IOException {
    if (!shouldRespond) {
        return null;
    }
    ServerErrorCode serverError = hardError != null ? hardError : serverErrors.size() > 0 ? serverErrors.poll() : ServerErrorCode.No_Error;
    RequestOrResponseType type = ((RequestOrResponse) send).getRequestType();
    RequestOrResponse response;
    requestCounts.computeIfAbsent(type, k -> new LongAdder()).increment();
    switch(type) {
        case PutRequest:
            response = makePutResponse((PutRequest) send, serverError);
            break;
        case GetRequest:
            response = makeGetResponse((GetRequest) send, serverError);
            break;
        case DeleteRequest:
            response = makeDeleteResponse((DeleteRequest) send, serverError);
            break;
        case TtlUpdateRequest:
            response = makeTtlUpdateResponse((TtlUpdateRequest) send, serverError);
            break;
        case UndeleteRequest:
            response = makeUndeleteResponse((UndeleteRequest) send, serverError);
            break;
        default:
            throw new IOException("Unknown request type received");
    }
    ByteBufferChannel channel = new ByteBufferChannel(ByteBuffer.allocate((int) response.sizeInBytes()));
    response.writeTo(channel);
    response.release();
    ByteBuffer payload = channel.getBuffer();
    payload.flip();
    BoundedNettyByteBufReceive receive = new BoundedNettyByteBufReceive(100 * 1024 * 1024);
    receive.readFrom(Channels.newChannel(new ByteBufferInputStream(payload)));
    return receive;
}

Example 3 with UndeleteRequest

use of com.github.ambry.protocol.UndeleteRequest in project ambry by linkedin.

the class ServerTestUtil method undeleteCornerCasesTest.

static void undeleteCornerCasesTest(MockCluster cluster, PortType portType, SSLConfig clientSSLConfig1, SSLConfig clientSSLConfig2, SSLConfig clientSSLConfig3, SSLSocketFactory clientSSLSocketFactory1, SSLSocketFactory clientSSLSocketFactory2, SSLSocketFactory clientSSLSocketFactory3, MockNotificationSystem notificationSystem, Properties routerProps, boolean testEncryption) {
    MockClusterMap clusterMap = cluster.getClusterMap();
    byte[] userMetadata = new byte[1000];
    byte[] data = new byte[31870];
    byte[] encryptionKey = new byte[100];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties properties = new BlobProperties(31870, "serviceid1", accountId, containerId, testEncryption, cluster.time.milliseconds());
    TestUtils.RANDOM.nextBytes(userMetadata);
    TestUtils.RANDOM.nextBytes(data);
    if (testEncryption) {
        TestUtils.RANDOM.nextBytes(encryptionKey);
    }
    short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
    Map<String, List<DataNodeId>> dataNodesPerDC = clusterMap.getDataNodes().stream().collect(Collectors.groupingBy(DataNodeId::getDatacenterName));
    Map<String, Pair<SSLConfig, SSLSocketFactory>> sslSettingPerDC = new HashMap<>();
    sslSettingPerDC.put("DC1", new Pair<>(clientSSLConfig1, clientSSLSocketFactory1));
    sslSettingPerDC.put("DC2", new Pair<>(clientSSLConfig2, clientSSLSocketFactory2));
    sslSettingPerDC.put("DC3", new Pair<>(clientSSLConfig3, clientSSLSocketFactory3));
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
    DataNodeId dataNodeId = dataNodesPerDC.get("DC1").get(0);
    Router router = null;
    try {
        Properties routerProperties = getRouterProps("DC1");
        routerProperties.putAll(routerProps);
        VerifiableProperties routerVerifiableProps = new VerifiableProperties(routerProperties);
        AccountService accountService = new InMemAccountService(false, true);
        router = new NonBlockingRouterFactory(routerVerifiableProps, clusterMap, new MockNotificationSystem(clusterMap), getSSLFactoryIfRequired(routerVerifiableProps), accountService).getRouter();
        // channels to all datanodes
        List<ConnectedChannel> channels = new ArrayList<>();
        for (Map.Entry<String, List<DataNodeId>> entry : dataNodesPerDC.entrySet()) {
            Pair<SSLConfig, SSLSocketFactory> pair = sslSettingPerDC.get(entry.getKey());
            for (DataNodeId node : entry.getValue()) {
                ConnectedChannel connectedChannel = getBlockingChannelBasedOnPortType(portType, node, pair.getSecond(), pair.getFirst());
                connectedChannel.connect();
                channels.add(connectedChannel);
            }
        }
        // ////////////////////////////////////////////////////
        // Corner case 1: When only one datacenter has delete
        // ////////////////////////////////////////////////////
        BlobId blobId1 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        ConnectedChannel channel = getBlockingChannelBasedOnPortType(portType, dataNodeId, clientSSLSocketFactory1, clientSSLConfig1);
        channel.connect();
        PutRequest putRequest = new PutRequest(1, "client1", blobId1, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        DataInputStream putResponseStream = channel.sendAndReceive(putRequest).getInputStream();
        PutResponse response = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response.getError());
        notificationSystem.awaitBlobCreations(blobId1.toString());
        // Now stop the replications this partition.
        PartitionId partitionId = blobId1.getPartition();
        controlReplicationForPartition(channels, partitionId, false);
        // Now send the delete to two data nodes in the same DC
        List<DataNodeId> toBeDeleteDataNodes = dataNodesPerDC.values().stream().findFirst().get();
        Pair<SSLConfig, SSLSocketFactory> pair = sslSettingPerDC.get(toBeDeleteDataNodes.get(0).getDatacenterName());
        ConnectedChannel channel1 = getBlockingChannelBasedOnPortType(portType, toBeDeleteDataNodes.get(0), pair.getSecond(), pair.getFirst());
        channel1.connect();
        ConnectedChannel channel2 = getBlockingChannelBasedOnPortType(portType, toBeDeleteDataNodes.get(1), pair.getSecond(), pair.getFirst());
        channel2.connect();
        DeleteRequest deleteRequest1 = new DeleteRequest(1, "deleteClient", blobId1, System.currentTimeMillis());
        DataInputStream stream = channel1.sendAndReceive(deleteRequest1).getInputStream();
        DeleteResponse deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        DeleteRequest deleteRequest2 = new DeleteRequest(1, "deleteClient", blobId1, deleteRequest1.getDeletionTimeInMs());
        stream = channel2.sendAndReceive(deleteRequest2).getInputStream();
        deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        // Now send the undelete operation through router, and it should fail because of not deleted error.
        Future<Void> future = router.undeleteBlob(blobId1.toString(), "service");
        try {
            future.get();
            fail("Undelete blob " + blobId1.toString() + " should fail");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof RouterException);
            assertEquals(RouterErrorCode.BlobNotDeleted, ((RouterException) e.getCause()).getErrorCode());
        }
        // Now see if either data node 1 or data node 2 has undelete or not, if so, undelete would replicate. If not,
        // delete would replicate.
        List<PartitionRequestInfo> partitionRequestInfoList = getPartitionRequestInfoListFromBlobId(blobId1);
        boolean hasUndelete = false;
        for (ConnectedChannel connectedChannel : new ConnectedChannel[] { channel1, channel2 }) {
            GetRequest getRequest = new GetRequest(1, "clientId1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.Include_All);
            stream = channel1.sendAndReceive(getRequest).getInputStream();
            GetResponse getResponse = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.No_Error, getResponse.getPartitionResponseInfoList().get(0).getErrorCode());
            MessageFormatRecord.deserializeBlobProperties(getResponse.getInputStream());
            hasUndelete = getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getLifeVersion() == (short) 1;
            if (hasUndelete) {
                break;
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // Now restart the replication
        controlReplicationForPartition(channels, partitionId, true);
        if (hasUndelete) {
            notificationSystem.awaitBlobUndeletes(blobId1.toString());
        } else {
            notificationSystem.awaitBlobDeletions(blobId1.toString());
        }
        for (ConnectedChannel connectedChannel : channels) {
            GetRequest getRequest = new GetRequest(1, "clientId1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.Include_All);
            stream = connectedChannel.sendAndReceive(getRequest).getInputStream();
            GetResponse getResponse = GetResponse.readFrom(stream, clusterMap);
            releaseNettyBufUnderneathStream(stream);
            assertEquals(ServerErrorCode.No_Error, getResponse.getPartitionResponseInfoList().get(0).getErrorCode());
            MessageFormatRecord.deserializeBlobProperties(getResponse.getInputStream());
            if (hasUndelete) {
                assertEquals((short) 1, getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getLifeVersion());
                assertTrue(getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).isUndeleted());
                assertFalse(getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).isDeleted());
            } else {
                assertEquals((short) 0, getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getLifeVersion());
                assertTrue(getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).isDeleted());
                assertFalse(getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).isUndeleted());
            }
        }
        // ///////////////////////////////////////////////////////////
        // Corner case 2: two data nodes have different life versions
        // //////////////////////////////////////////////////////////
        BlobId blobId2 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        putRequest = new PutRequest(1, "client1", blobId2, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest).getInputStream();
        response = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response.getError());
        notificationSystem.awaitBlobCreations(blobId2.toString());
        // Now delete this blob on all servers.
        DeleteRequest deleteRequest = new DeleteRequest(1, "deleteClient", blobId2, System.currentTimeMillis());
        stream = channel.sendAndReceive(deleteRequest).getInputStream();
        deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        notificationSystem.awaitBlobDeletions(blobId2.toString());
        // Now stop the replication
        partitionId = blobId2.getPartition();
        controlReplicationForPartition(channels, partitionId, false);
        // Now send the undelete to two data nodes in the same DC and then send delete
        UndeleteRequest undeleteRequest = new UndeleteRequest(1, "undeleteClient", blobId2, System.currentTimeMillis());
        stream = channel1.sendAndReceive(undeleteRequest).getInputStream();
        UndeleteResponse undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, undeleteResponse.getError());
        assertEquals((short) 1, undeleteResponse.getLifeVersion());
        undeleteRequest = new UndeleteRequest(1, "undeleteClient", blobId2, undeleteRequest.getOperationTimeMs());
        stream = channel2.sendAndReceive(undeleteRequest).getInputStream();
        undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, undeleteResponse.getError());
        assertEquals((short) 1, undeleteResponse.getLifeVersion());
        deleteRequest1 = new DeleteRequest(1, "deleteClient", blobId2, System.currentTimeMillis());
        stream = channel1.sendAndReceive(deleteRequest1).getInputStream();
        deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        deleteRequest2 = new DeleteRequest(1, "deleteClient", blobId2, deleteRequest1.getDeletionTimeInMs());
        stream = channel2.sendAndReceive(deleteRequest2).getInputStream();
        deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        // Now send the undelete operation through router, and it should fail because of lifeVersion conflict error.
        future = router.undeleteBlob(blobId2.toString(), "service");
        try {
            future.get();
            fail("Undelete blob " + blobId2.toString() + " should fail");
        } catch (ExecutionException e) {
            assertTrue(e.getCause() instanceof RouterException);
            assertEquals(RouterErrorCode.LifeVersionConflict, ((RouterException) e.getCause()).getErrorCode());
        }
        // Now restart the replication
        controlReplicationForPartition(channels, partitionId, true);
        notificationSystem.awaitBlobUndeletes(blobId2.toString());
        // Now after replication is resumed, the undelete of lifeversion 2 will eventually be replicated to all servers.
        partitionRequestInfoList = getPartitionRequestInfoListFromBlobId(blobId2);
        for (ConnectedChannel connectedChannel : channels) {
            // Even if the notificationSystem acknowledged the undelete, it might be triggered by undelete at lifeversion 1.
            // So check in a loop with a time out.
            long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
            while (true) {
                GetRequest getRequest = new GetRequest(1, "clientId1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.Include_All);
                stream = connectedChannel.sendAndReceive(getRequest).getInputStream();
                GetResponse getResponse = GetResponse.readFrom(stream, clusterMap);
                assertEquals(ServerErrorCode.No_Error, getResponse.getPartitionResponseInfoList().get(0).getErrorCode());
                MessageFormatRecord.deserializeBlobProperties(getResponse.getInputStream());
                if (getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getLifeVersion() == 2) {
                    assertTrue(getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).isUndeleted());
                    assertFalse(getResponse.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).isDeleted());
                    break;
                } else {
                    Thread.sleep(1000);
                    if (System.currentTimeMillis() > deadline) {
                        throw new TimeoutException("Fail to get blob " + blobId2 + " at lifeversion 2 at " + connectedChannel.getRemoteHost());
                    }
                }
            }
        }
        releaseNettyBufUnderneathStream(stream);
        for (ConnectedChannel connectedChannel : channels) {
            connectedChannel.disconnect();
        }
        channel1.disconnect();
        channel2.disconnect();
        channel.disconnect();
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    } finally {
        if (router != null) {
            try {
                router.close();
            } catch (Exception e) {
            }
        }
    }
}

Example 4 with UndeleteRequest

use of com.github.ambry.protocol.UndeleteRequest in project ambry by linkedin.

the class ServerTestUtil method endToEndTest.

static void endToEndTest(Port targetPort, String routerDatacenter, MockCluster cluster, SSLConfig clientSSLConfig, SSLSocketFactory clientSSLSocketFactory, Properties routerProps, boolean testEncryption) {
    try {
        MockClusterMap clusterMap = cluster.getClusterMap();
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        byte[] userMetadata = new byte[1000];
        byte[] data = new byte[31870];
        byte[] encryptionKey = new byte[100];
        short accountId = Utils.getRandomShort(TestUtils.RANDOM);
        short containerId = Utils.getRandomShort(TestUtils.RANDOM);
        BlobProperties properties = new BlobProperties(31870, "serviceid1", accountId, containerId, testEncryption, cluster.time.milliseconds());
        TestUtils.RANDOM.nextBytes(userMetadata);
        TestUtils.RANDOM.nextBytes(data);
        if (testEncryption) {
            TestUtils.RANDOM.nextBytes(encryptionKey);
        }
        List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
        short blobIdVersion = CommonTestUtils.getCurrentBlobIdVersion();
        BlobId blobId1 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId2 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId3 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId4 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        BlobId blobId5 = new BlobId(blobIdVersion, BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partitionIds.get(0), false, BlobId.BlobDataType.DATACHUNK);
        // put blob 1
        PutRequest putRequest = new PutRequest(1, "client1", blobId1, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        ConnectedChannel channel = getBlockingChannelBasedOnPortType(targetPort, "localhost", clientSSLSocketFactory, clientSSLConfig);
        channel.connect();
        DataInputStream putResponseStream = channel.sendAndReceive(putRequest).getInputStream();
        PutResponse response = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response.getError());
        // put blob 2 with an expiry time and apply TTL update later
        BlobProperties propertiesForTtlUpdate = new BlobProperties(31870, "serviceid1", "ownerid", "image/png", false, TestUtils.TTL_SECS, cluster.time.milliseconds(), accountId, containerId, testEncryption, null, null, null);
        long ttlUpdateBlobExpiryTimeMs = getExpiryTimeMs(propertiesForTtlUpdate);
        PutRequest putRequest2 = new PutRequest(1, "client1", blobId2, propertiesForTtlUpdate, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest2).getInputStream();
        PutResponse response2 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response2.getError());
        // put blob 3
        PutRequest putRequest3 = new PutRequest(1, "client1", blobId3, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest3).getInputStream();
        PutResponse response3 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response3.getError());
        // put blob 4 that is expired
        BlobProperties propertiesExpired = new BlobProperties(31870, "serviceid1", "ownerid", "jpeg", false, 0, cluster.time.milliseconds(), accountId, containerId, testEncryption, null, null, null);
        PutRequest putRequest4 = new PutRequest(1, "client1", blobId4, propertiesExpired, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest4).getInputStream();
        PutResponse response4 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals(ServerErrorCode.No_Error, response4.getError());
        cluster.time.sleep(10000);
        // get blob properties
        ArrayList<BlobId> ids = new ArrayList<>();
        MockPartitionId partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(blobId1);
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        DataInputStream stream = channel.sendAndReceive(getRequest1).getInputStream();
        GetResponse resp1 = GetResponse.readFrom(stream, clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            releaseNettyBufUnderneathStream(stream);
        } catch (MessageFormatException e) {
            fail();
        }
        // get blob properties with expired flag set
        ids = new ArrayList<BlobId>();
        partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.Include_Expired_Blobs);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            releaseNettyBufUnderneathStream(stream);
        } catch (MessageFormatException e) {
            fail();
        }
        // get blob properties for expired blob
        // 1. With no flag
        ArrayList<BlobId> idsExpired = new ArrayList<>();
        MockPartitionId partitionExpired = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        idsExpired.add(blobId4);
        ArrayList<PartitionRequestInfo> partitionRequestInfoListExpired = new ArrayList<>();
        PartitionRequestInfo partitionRequestInfoExpired = new PartitionRequestInfo(partitionExpired, idsExpired);
        partitionRequestInfoListExpired.add(partitionRequestInfoExpired);
        GetRequest getRequestExpired = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoListExpired, GetOption.None);
        DataInputStream streamExpired = channel.sendAndReceive(getRequestExpired).getInputStream();
        GetResponse respExpired = GetResponse.readFrom(streamExpired, clusterMap);
        assertEquals(ServerErrorCode.Blob_Expired, respExpired.getPartitionResponseInfoList().get(0).getErrorCode());
        releaseNettyBufUnderneathStream(streamExpired);
        // 2. With Include_Expired flag
        idsExpired = new ArrayList<>();
        partitionExpired = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        idsExpired.add(blobId4);
        partitionRequestInfoListExpired = new ArrayList<>();
        partitionRequestInfoExpired = new PartitionRequestInfo(partitionExpired, idsExpired);
        partitionRequestInfoListExpired.add(partitionRequestInfoExpired);
        getRequestExpired = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoListExpired, GetOption.Include_Expired_Blobs);
        streamExpired = channel.sendAndReceive(getRequestExpired).getInputStream();
        respExpired = GetResponse.readFrom(streamExpired, clusterMap);
        try {
            BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(respExpired.getInputStream());
            assertEquals(31870, propertyOutput.getBlobSize());
            assertEquals("serviceid1", propertyOutput.getServiceId());
            assertEquals("ownerid", propertyOutput.getOwnerId());
            assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
            assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            releaseNettyBufUnderneathStream(streamExpired);
        } catch (MessageFormatException e) {
            fail();
        }
        // get user metadata
        GetRequest getRequest2 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest2).getInputStream();
        GetResponse resp2 = GetResponse.readFrom(stream, clusterMap);
        try {
            ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp2.getInputStream());
            assertArrayEquals(userMetadata, userMetadataOutput.array());
            if (testEncryption) {
                assertNotNull("MessageMetadata should not have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
            } else {
                assertNull("MessageMetadata should have been null", resp2.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            }
            releaseNettyBufUnderneathStream(stream);
        } catch (MessageFormatException e) {
            fail();
        }
        // get blob info
        GetRequest getRequest3 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobInfo, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest3).getInputStream();
        GetResponse resp3 = GetResponse.readFrom(stream, clusterMap);
        InputStream responseStream = resp3.getInputStream();
        // verify blob properties.
        BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(responseStream);
        assertEquals(31870, propertyOutput.getBlobSize());
        assertEquals("serviceid1", propertyOutput.getServiceId());
        assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
        assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
        // verify user metadata
        ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(responseStream);
        assertArrayEquals(userMetadata, userMetadataOutput.array());
        if (testEncryption) {
            assertNotNull("MessageMetadata should not have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
            assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
        } else {
            assertNull("MessageMetadata should have been null", resp3.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob all
        GetRequest getRequest4 = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest4).getInputStream();
        GetResponse resp4 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp4.getInputStream();
        BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        // verify content
        assertArrayEquals("Content mismatch.", data, actualBlobData);
        if (testEncryption) {
            assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
            assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
        } else {
            assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
        }
        releaseNettyBufUnderneathStream(stream);
        // encryptionKey in this test doesn't have any relation to the content. Both are random bytes for test purposes.
        if (!testEncryption) {
            // Use router to get the blob
            Properties routerProperties = getRouterProps(routerDatacenter);
            routerProperties.putAll(routerProps);
            VerifiableProperties routerVerifiableProps = new VerifiableProperties(routerProperties);
            AccountService accountService = new InMemAccountService(false, true);
            Router router = new NonBlockingRouterFactory(routerVerifiableProps, clusterMap, new MockNotificationSystem(clusterMap), getSSLFactoryIfRequired(routerVerifiableProps), accountService).getRouter();
            checkBlobId(router, blobId1, data);
            router.close();
        }
        checkTtlUpdateStatus(channel, clusterMap, blobIdFactory, blobId2, data, false, ttlUpdateBlobExpiryTimeMs);
        updateBlobTtl(channel, blobId2, cluster.time.milliseconds());
        checkTtlUpdateStatus(channel, clusterMap, blobIdFactory, blobId2, data, true, Utils.Infinite_Time);
        // fetch blob that does not exist
        // get blob properties
        ids = new ArrayList<>();
        partition = (MockPartitionId) clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, clusterMap.getLocalDatacenterId(), properties.getAccountId(), properties.getContainerId(), partition, false, BlobId.BlobDataType.DATACHUNK));
        partitionRequestInfoList.clear();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest5 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest5).getInputStream();
        GetResponse resp5 = GetResponse.readFrom(stream, clusterMap);
        assertEquals(ServerErrorCode.Blob_Not_Found, resp5.getPartitionResponseInfoList().get(0).getErrorCode());
        releaseNettyBufUnderneathStream(stream);
        // stop the store via AdminRequest
        System.out.println("Begin to stop a BlobStore");
        AdminRequest adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionIds.get(0), 1, "clientid2");
        BlobStoreControlAdminRequest controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StopStore, adminRequest);
        stream = channel.sendAndReceive(controlRequest).getInputStream();
        AdminResponse adminResponse = AdminResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Stop store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
        // put a blob on a stopped store, which should fail
        putRequest = new PutRequest(1, "client1", blobId5, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest).getInputStream();
        response = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals("Put blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, response.getError());
        // get a blob properties on a stopped store, which should fail
        ids = new ArrayList<>();
        partition = (MockPartitionId) blobId1.getPartition();
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partition, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        assertEquals("Get blob properties on stopped store should fail", ServerErrorCode.Replica_Unavailable, resp1.getPartitionResponseInfoList().get(0).getErrorCode());
        releaseNettyBufUnderneathStream(stream);
        // delete a blob on a stopped store, which should fail
        DeleteRequest deleteRequest = new DeleteRequest(1, "deleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(deleteRequest).getInputStream();
        DeleteResponse deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Delete blob on stopped store should fail", ServerErrorCode.Replica_Unavailable, deleteResponse.getError());
        // start the store via AdminRequest
        System.out.println("Begin to restart the BlobStore");
        adminRequest = new AdminRequest(AdminRequestOrResponseType.BlobStoreControl, partitionIds.get(0), 1, "clientid2");
        controlRequest = new BlobStoreControlAdminRequest((short) 0, BlobStoreControlAction.StartStore, adminRequest);
        stream = channel.sendAndReceive(controlRequest).getInputStream();
        adminResponse = AdminResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Start store admin request should succeed", ServerErrorCode.No_Error, adminResponse.getError());
        List<? extends ReplicaId> replicaIds = partitionIds.get(0).getReplicaIds();
        for (ReplicaId replicaId : replicaIds) {
            // forcibly mark replicas and disks as up.
            MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
            mockReplicaId.markReplicaDownStatus(false);
            ((MockDiskId) mockReplicaId.getDiskId()).setDiskState(HardwareState.AVAILABLE, false);
        }
        // put a blob on a restarted store , which should succeed
        PutRequest putRequest5 = new PutRequest(1, "client1", blobId5, properties, ByteBuffer.wrap(userMetadata), Unpooled.wrappedBuffer(data), properties.getBlobSize(), BlobType.DataBlob, testEncryption ? ByteBuffer.wrap(encryptionKey) : null);
        putResponseStream = channel.sendAndReceive(putRequest5).getInputStream();
        PutResponse response5 = PutResponse.readFrom(putResponseStream);
        releaseNettyBufUnderneathStream(putResponseStream);
        assertEquals("Put blob on restarted store should succeed", ServerErrorCode.No_Error, response5.getError());
        // get a blob on a restarted store , which should succeed
        ids = new ArrayList<>();
        PartitionId partitionId = clusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0);
        ids.add(blobId1);
        partitionRequestInfoList = new ArrayList<>();
        partitionRequestInfo = new PartitionRequestInfo(partitionId, ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp1.getInputStream();
        blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        assertArrayEquals("Content mismatch.", data, actualBlobData);
        releaseNettyBufUnderneathStream(stream);
        // undelete a not-deleted blob should return fail
        UndeleteRequest undeleteRequest = new UndeleteRequest(1, "undeleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(undeleteRequest).getInputStream();
        UndeleteResponse undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Undelete blob should succeed", ServerErrorCode.Blob_Not_Deleted, undeleteResponse.getError());
        // delete a blob on a restarted store , which should succeed
        deleteRequest = new DeleteRequest(1, "deleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(deleteRequest).getInputStream();
        deleteResponse = DeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Delete blob on restarted store should succeed", ServerErrorCode.No_Error, deleteResponse.getError());
        // undelete a deleted blob, which should succeed
        undeleteRequest = new UndeleteRequest(2, "undeleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(undeleteRequest).getInputStream();
        undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Undelete blob should succeed", ServerErrorCode.No_Error, undeleteResponse.getError());
        assertEquals("Undelete life version mismatch", undeleteResponse.getLifeVersion(), (short) 1);
        // undelete an already undeleted blob, which should fail
        undeleteRequest = new UndeleteRequest(3, "undeleteClient", blobId1, System.currentTimeMillis());
        stream = channel.sendAndReceive(undeleteRequest).getInputStream();
        undeleteResponse = UndeleteResponse.readFrom(stream);
        releaseNettyBufUnderneathStream(stream);
        assertEquals("Undelete blob should fail", ServerErrorCode.Blob_Already_Undeleted, undeleteResponse.getError());
        assertEquals("LifeVersion Mismatch", (short) 1, undeleteResponse.getLifeVersion());
        // get an undeleted blob, which should succeed
        getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp1.getInputStream();
        blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        assertArrayEquals("Content mismatch", data, actualBlobData);
        releaseNettyBufUnderneathStream(stream);
        // Bounce servers to make them read the persisted token file.
        cluster.stopServers();
        cluster.reinitServers();
        channel.disconnect();
        channel.connect();
        // get an undeleted blob after restart, which should succeed
        getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel.sendAndReceive(getRequest1).getInputStream();
        resp1 = GetResponse.readFrom(stream, clusterMap);
        responseStream = resp1.getInputStream();
        blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, blobIdFactory);
        actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
        assertArrayEquals("Content mismatch", data, actualBlobData);
        releaseNettyBufUnderneathStream(stream);
        channel.disconnect();
    } catch (Exception e) {
        e.printStackTrace();
        assertNull(e);
    } finally {
        List<? extends ReplicaId> replicaIds = cluster.getClusterMap().getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS).get(0).getReplicaIds();
        for (ReplicaId replicaId : replicaIds) {
            MockReplicaId mockReplicaId = (MockReplicaId) replicaId;
            ((MockDiskId) mockReplicaId.getDiskId()).setDiskState(HardwareState.AVAILABLE, true);
        }
    }
}

Example 5 with UndeleteRequest

use of com.github.ambry.protocol.UndeleteRequest in project ambry by linkedin.

the class ServerTestUtil method endToEndReplicationWithMultiNodeMultiPartitionTest.

static void endToEndReplicationWithMultiNodeMultiPartitionTest(int interestedDataNodePortNumber, Port dataNode1Port, Port dataNode2Port, Port dataNode3Port, MockCluster cluster, SSLConfig clientSSLConfig1, SSLConfig clientSSLConfig2, SSLConfig clientSSLConfig3, SSLSocketFactory clientSSLSocketFactory1, SSLSocketFactory clientSSLSocketFactory2, SSLSocketFactory clientSSLSocketFactory3, MockNotificationSystem notificationSystem, boolean testEncryption) throws Exception {
    // interestedDataNodePortNumber is used to locate the datanode and hence has to be PlainTextPort
    MockClusterMap clusterMap = cluster.getClusterMap();
    BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
    List<AmbryServer> serverList = cluster.getServers();
    byte[] usermetadata = new byte[100];
    byte[] data = new byte[100];
    byte[] encryptionKey = null;
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties properties = new BlobProperties(100, "serviceid1", null, null, false, TestUtils.TTL_SECS, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
    long expectedExpiryTimeMs = getExpiryTimeMs(properties);
    TestUtils.RANDOM.nextBytes(usermetadata);
    TestUtils.RANDOM.nextBytes(data);
    if (testEncryption) {
        encryptionKey = new byte[100];
        TestUtils.RANDOM.nextBytes(encryptionKey);
    }
    // connect to all the servers
    ConnectedChannel channel1 = getBlockingChannelBasedOnPortType(dataNode1Port, "localhost", clientSSLSocketFactory1, clientSSLConfig1);
    ConnectedChannel channel2 = getBlockingChannelBasedOnPortType(dataNode2Port, "localhost", clientSSLSocketFactory2, clientSSLConfig2);
    ConnectedChannel channel3 = getBlockingChannelBasedOnPortType(dataNode3Port, "localhost", clientSSLSocketFactory3, clientSSLConfig3);
    // put all the blobs to random servers
    channel1.connect();
    channel2.connect();
    channel3.connect();
    int noOfParallelThreads = 3;
    int totalBlobsToPut = 50;
    CountDownLatch latch = new CountDownLatch(noOfParallelThreads);
    List<DirectSender> runnables = new ArrayList<DirectSender>(noOfParallelThreads);
    ConnectedChannel channel = null;
    for (int i = 0; i < noOfParallelThreads; i++) {
        if (i % noOfParallelThreads == 0) {
            channel = channel1;
        } else if (i % noOfParallelThreads == 1) {
            channel = channel2;
        } else if (i % noOfParallelThreads == 2) {
            channel = channel3;
        }
        DirectSender runnable = new DirectSender(cluster, channel, totalBlobsToPut, data, usermetadata, properties, encryptionKey, latch);
        runnables.add(runnable);
        Thread threadToRun = new Thread(runnable);
        threadToRun.start();
    }
    assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
    // wait till replication can complete
    List<BlobId> blobIds = new ArrayList<BlobId>();
    for (int i = 0; i < runnables.size(); i++) {
        blobIds.addAll(runnables.get(i).getBlobIds());
    }
    for (BlobId blobId : blobIds) {
        notificationSystem.awaitBlobCreations(blobId.getID());
    }
    // Now that the blob is created and replicated, test the cases where a put request arrives for the same blob id
    // later than replication.
    testLatePutRequest(blobIds.get(0), properties, usermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.No_Error);
    // Test the case where a put arrives with the same id as one in the server, but the blob is not identical.
    BlobProperties differentProperties = new BlobProperties(properties.getBlobSize(), properties.getServiceId(), accountId, containerId, testEncryption, cluster.time.milliseconds());
    testLatePutRequest(blobIds.get(0), differentProperties, usermetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
    byte[] differentUserMetadata = Arrays.copyOf(usermetadata, usermetadata.length);
    differentUserMetadata[0] = (byte) ~differentUserMetadata[0];
    testLatePutRequest(blobIds.get(0), properties, differentUserMetadata, data, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
    byte[] differentData = Arrays.copyOf(data, data.length);
    differentData[0] = (byte) ~differentData[0];
    testLatePutRequest(blobIds.get(0), properties, usermetadata, differentData, encryptionKey, channel1, channel2, channel3, ServerErrorCode.Blob_Already_Exists);
    // verify blob properties, metadata and blob across all nodes
    for (int i = 0; i < 3; i++) {
        channel = null;
        if (i == 0) {
            channel = channel1;
        } else if (i == 1) {
            channel = channel2;
        } else if (i == 2) {
            channel = channel3;
        }
        ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
        for (int j = 0; j < blobIds.size(); j++) {
            ArrayList<BlobId> ids = new ArrayList<BlobId>();
            ids.add(blobIds.get(j));
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
            DataInputStream stream = channel.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            try {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                assertEquals(100, propertyOutput.getBlobSize());
                assertEquals("serviceid1", propertyOutput.getServiceId());
                assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                assertEquals("Expiration time mismatch (props)", expectedExpiryTimeMs, getExpiryTimeMs(propertyOutput));
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                fail();
            }
            // get user metadata
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
            stream = channel.sendAndReceive(getRequest).getInputStream();
            resp = GetResponse.readFrom(stream, clusterMap);
            try {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                assertArrayEquals(usermetadata, userMetadataOutput.array());
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                e.printStackTrace();
                fail();
            }
            releaseNettyBufUnderneathStream(stream);
            // get blob
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
            stream = channel.sendAndReceive(getRequest).getInputStream();
            resp = GetResponse.readFrom(stream, clusterMap);
            try {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                byte[] blobout = getBlobDataAndRelease(blobData);
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                e.printStackTrace();
                fail();
            }
            releaseNettyBufUnderneathStream(stream);
            // get blob all
            getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
            stream = channel.sendAndReceive(getRequest).getInputStream();
            resp = GetResponse.readFrom(stream, clusterMap);
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                assertEquals("Expiration time mismatch (props)", expectedExpiryTimeMs, getExpiryTimeMs(blobAll.getBlobInfo().getBlobProperties()));
                assertEquals("Expiration time mismatch (MessageInfo)", expectedExpiryTimeMs, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                e.printStackTrace();
                fail();
            }
        }
    }
    // ttl update all blobs and wait for replication
    Map<ConnectedChannel, List<BlobId>> channelToBlobIds = new HashMap<>();
    for (int i = 0; i < blobIds.size(); i++) {
        final BlobId blobId = blobIds.get(i);
        if (i % 3 == 0) {
            channelToBlobIds.computeIfAbsent(channel1, updateChannel -> new ArrayList<>()).add(blobId);
        } else if (i % 3 == 1) {
            channelToBlobIds.computeIfAbsent(channel2, updateChannel -> new ArrayList<>()).add(blobId);
        } else {
            channelToBlobIds.computeIfAbsent(channel3, updateChannel -> new ArrayList<>()).add(blobId);
        }
    }
    channelToBlobIds.entrySet().stream().map(entry -> CompletableFuture.supplyAsync(() -> {
        try {
            for (BlobId blobId : entry.getValue()) {
                updateBlobTtl(entry.getKey(), blobId, cluster.time.milliseconds());
            }
            return null;
        } catch (Throwable e) {
            throw new RuntimeException("Exception updating ttl for: " + entry, e);
        }
    })).forEach(CompletableFuture::join);
    // check that the TTL update has propagated
    blobIds.forEach(blobId -> notificationSystem.awaitBlobUpdates(blobId.getID(), UpdateType.TTL_UPDATE));
    // check all servers
    for (ConnectedChannel channelToUse : new ConnectedChannel[] { channel1, channel2, channel3 }) {
        for (BlobId blobId : blobIds) {
            checkTtlUpdateStatus(channelToUse, clusterMap, blobIdFactory, blobId, data, true, Utils.Infinite_Time);
        }
    }
    // delete random blobs, wait for replication and ensure it is deleted in all nodes
    Set<BlobId> blobsDeleted = new HashSet<BlobId>();
    Set<BlobId> blobsChecked = new HashSet<BlobId>();
    for (int i = 0; i < blobIds.size(); i++) {
        int j = new Random().nextInt(3);
        if (j == 0) {
            j = new Random().nextInt(3);
            if (j == 0) {
                channel = channel1;
            } else if (j == 1) {
                channel = channel2;
            } else if (j == 2) {
                channel = channel3;
            }
            DeleteRequest deleteRequest = new DeleteRequest(1, "reptest", blobIds.get(i), System.currentTimeMillis());
            DataInputStream deleteResponseStream = channel.sendAndReceive(deleteRequest).getInputStream();
            DeleteResponse deleteResponse = DeleteResponse.readFrom(deleteResponseStream);
            releaseNettyBufUnderneathStream(deleteResponseStream);
            assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
            blobsDeleted.add(blobIds.get(i));
        }
    }
    Iterator<BlobId> iterator = blobsDeleted.iterator();
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    while (iterator.hasNext()) {
        BlobId deletedId = iterator.next();
        notificationSystem.awaitBlobDeletions(deletedId.getID());
        for (int j = 0; j < 3; j++) {
            if (j == 0) {
                channel = channel1;
            } else if (j == 1) {
                channel = channel2;
            } else if (j == 2) {
                channel = channel3;
            }
            ArrayList<BlobId> ids = new ArrayList<BlobId>();
            ids.add(deletedId);
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(deletedId.getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
            DataInputStream stream = channel.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.Blob_Deleted, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            releaseNettyBufUnderneathStream(stream);
        }
    }
    // take a server down, clean up a mount path, start and ensure replication fixes it
    serverList.get(0).shutdown();
    serverList.get(0).awaitShutdown();
    MockDataNodeId dataNode = (MockDataNodeId) clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
    System.out.println("Cleaning mount path " + dataNode.getMountPaths().get(0));
    for (ReplicaId replicaId : clusterMap.getReplicaIds(dataNode)) {
        if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(0)) == 0) {
            System.out.println("Cleaning partition " + replicaId.getPartitionId());
        }
    }
    deleteFolderContent(new File(dataNode.getMountPaths().get(0)), false);
    for (int i = 0; i < blobIds.size(); i++) {
        for (ReplicaId replicaId : blobIds.get(i).getPartition().getReplicaIds()) {
            if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(0)) == 0) {
                if (blobsDeleted.contains(blobIds.get(i))) {
                    notificationSystem.decrementDeletedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
                } else {
                    notificationSystem.decrementCreatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
                    notificationSystem.decrementUpdatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort(), UpdateType.TTL_UPDATE);
                }
            }
        }
    }
    cluster.reinitServer(0);
    channel1.disconnect();
    channel1.connect();
    for (int j = 0; j < blobIds.size(); j++) {
        if (blobsDeleted.contains(blobIds.get(j))) {
            notificationSystem.awaitBlobDeletions(blobIds.get(j).getID());
        } else {
            notificationSystem.awaitBlobCreations(blobIds.get(j).getID());
            notificationSystem.awaitBlobUpdates(blobIds.get(j).getID(), UpdateType.TTL_UPDATE);
        }
        ArrayList<BlobId> ids = new ArrayList<BlobId>();
        ids.add(blobIds.get(j));
        partitionRequestInfoList.clear();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
        GetResponse resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
        } else {
            try {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                assertEquals(100, propertyOutput.getBlobSize());
                assertEquals("serviceid1", propertyOutput.getServiceId());
                assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get user metadata
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
        } else {
            try {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                assertArrayEquals(usermetadata, userMetadataOutput.array());
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
        } else {
            try {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                byte[] blobout = getBlobDataAndRelease(blobData);
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob all
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsDeleted.contains(blobIds.get(j)));
            blobsDeleted.remove(blobIds.get(j));
            blobsChecked.add(blobIds.get(j));
        } else {
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
    }
    assertEquals(0, blobsDeleted.size());
    // take a server down, clean all contents, start and ensure replication fixes it
    serverList.get(0).shutdown();
    serverList.get(0).awaitShutdown();
    dataNode = (MockDataNodeId) clusterMap.getDataNodeId("localhost", interestedDataNodePortNumber);
    for (int i = 0; i < dataNode.getMountPaths().size(); i++) {
        System.out.println("Cleaning mount path " + dataNode.getMountPaths().get(i));
        for (ReplicaId replicaId : clusterMap.getReplicaIds(dataNode)) {
            if (replicaId.getMountPath().compareToIgnoreCase(dataNode.getMountPaths().get(i)) == 0) {
                System.out.println("Cleaning partition " + replicaId.getPartitionId());
            }
        }
        deleteFolderContent(new File(dataNode.getMountPaths().get(i)), false);
    }
    for (int i = 0; i < blobIds.size(); i++) {
        if (blobsChecked.contains(blobIds.get(i))) {
            notificationSystem.decrementDeletedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
        } else {
            notificationSystem.decrementCreatedReplica(blobIds.get(i).getID(), dataNode.getHostname(), dataNode.getPort());
        }
    }
    cluster.reinitServer(0);
    channel1.disconnect();
    channel1.connect();
    for (int j = 0; j < blobIds.size(); j++) {
        if (blobsChecked.contains(blobIds.get(j))) {
            notificationSystem.awaitBlobDeletions(blobIds.get(j).getID());
        } else {
            notificationSystem.awaitBlobCreations(blobIds.get(j).getID());
        }
        ArrayList<BlobId> ids = new ArrayList<BlobId>();
        ids.add(blobIds.get(j));
        partitionRequestInfoList.clear();
        PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobIds.get(j).getPartition(), ids);
        partitionRequestInfoList.add(partitionRequestInfo);
        GetRequest getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
        DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
        GetResponse resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
        } else {
            try {
                BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp.getInputStream());
                assertEquals(100, propertyOutput.getBlobSize());
                assertEquals("serviceid1", propertyOutput.getServiceId());
                assertEquals("AccountId mismatch", accountId, propertyOutput.getAccountId());
                assertEquals("ContainerId mismatch", containerId, propertyOutput.getContainerId());
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get user metadata
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.BlobUserMetadata, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
        } else {
            try {
                ByteBuffer userMetadataOutput = MessageFormatRecord.deserializeUserMetadata(resp.getInputStream());
                assertArrayEquals(usermetadata, userMetadataOutput.array());
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.Blob, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
        } else {
            try {
                BlobData blobData = MessageFormatRecord.deserializeBlob(resp.getInputStream());
                byte[] blobout = getBlobDataAndRelease(blobData);
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("MessageMetadata should not have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0).getEncryptionKey().array());
                } else {
                    assertNull("MessageMetadata should have been null", resp.getPartitionResponseInfoList().get(0).getMessageMetadataList().get(0));
                }
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
        // get blob all
        getRequest = new GetRequest(1, "clientid2", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
        stream = channel1.sendAndReceive(getRequest).getInputStream();
        resp = GetResponse.readFrom(stream, clusterMap);
        if (resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Deleted || resp.getPartitionResponseInfoList().get(0).getErrorCode() == ServerErrorCode.Blob_Not_Found) {
            assertTrue(blobsChecked.contains(blobIds.get(j)));
            blobsChecked.remove(blobIds.get(j));
            blobsDeleted.add(blobIds.get(j));
        } else {
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
            } catch (MessageFormatException e) {
                fail();
            }
        }
        releaseNettyBufUnderneathStream(stream);
    }
    assertEquals(0, blobsChecked.size());
    short expectedLifeVersion = 1;
    for (int i = 0; i < 2; i++) {
        expectedLifeVersion += i;
        // First undelete all deleted blobs
        for (BlobId deletedId : blobsDeleted) {
            UndeleteRequest undeleteRequest = new UndeleteRequest(2, "reptest", deletedId, System.currentTimeMillis());
            DataInputStream undeleteResponseStream = channel3.sendAndReceive(undeleteRequest).getInputStream();
            UndeleteResponse undeleteResponse = UndeleteResponse.readFrom(undeleteResponseStream);
            releaseNettyBufUnderneathStream(undeleteResponseStream);
            assertEquals(ServerErrorCode.No_Error, undeleteResponse.getError());
            assertEquals(expectedLifeVersion, undeleteResponse.getLifeVersion());
        }
        Thread.sleep(5000);
        // Then use get request to get all the data back and make sure the lifeVersion is correct
        for (BlobId id : blobsDeleted) {
            // We don't need to wait for blob undeletes, since one of the hosts has Put Record deleted
            // from disk, so undelete this blob would end up replicating Put Record instead of undelete.
            // notificationSystem.awaitBlobUndeletes(id.toString());
            ArrayList<BlobId> ids = new ArrayList<>();
            ids.add(id);
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(id.getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            // get blob all
            GetRequest getRequest = new GetRequest(1, "clientid20", MessageFormatFlags.All, partitionRequestInfoList, GetOption.None);
            DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.No_Error, resp.getError());
            assertEquals(1, resp.getPartitionResponseInfoList().size());
            assertEquals(ServerErrorCode.No_Error, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            assertEquals(1, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().size());
            MessageInfo info = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
            assertEquals(expectedLifeVersion, info.getLifeVersion());
            assertFalse(info.isDeleted());
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                fail();
            }
        }
        for (BlobId id : blobsDeleted) {
            DeleteRequest deleteRequest = new DeleteRequest(1, "reptest", id, System.currentTimeMillis());
            DataInputStream deleteResponseStream = channel.sendAndReceive(deleteRequest).getInputStream();
            DeleteResponse deleteResponse = DeleteResponse.readFrom(deleteResponseStream);
            releaseNettyBufUnderneathStream(deleteResponseStream);
            assertEquals(ServerErrorCode.No_Error, deleteResponse.getError());
        }
        Thread.sleep(1000);
        for (BlobId id : blobsDeleted) {
            ArrayList<BlobId> ids = new ArrayList<>();
            ids.add(id);
            partitionRequestInfoList.clear();
            PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(id.getPartition(), ids);
            partitionRequestInfoList.add(partitionRequestInfo);
            // get blob all
            GetRequest getRequest = new GetRequest(1, "clientid200", MessageFormatFlags.All, partitionRequestInfoList, GetOption.Include_All);
            DataInputStream stream = channel1.sendAndReceive(getRequest).getInputStream();
            GetResponse resp = GetResponse.readFrom(stream, clusterMap);
            assertEquals(ServerErrorCode.No_Error, resp.getError());
            assertEquals(1, resp.getPartitionResponseInfoList().size());
            assertEquals(ServerErrorCode.No_Error, resp.getPartitionResponseInfoList().get(0).getErrorCode());
            assertEquals(1, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().size());
            MessageInfo info = resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
            assertEquals(expectedLifeVersion, info.getLifeVersion());
            assertTrue(info.isDeleted());
            try {
                BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(resp.getInputStream(), blobIdFactory);
                byte[] blobout = getBlobDataAndRelease(blobAll.getBlobData());
                assertArrayEquals(data, blobout);
                if (testEncryption) {
                    assertNotNull("EncryptionKey should not ne null", blobAll.getBlobEncryptionKey());
                    assertArrayEquals("EncryptionKey mismatch", encryptionKey, blobAll.getBlobEncryptionKey().array());
                } else {
                    assertNull("EncryptionKey should have been null", blobAll.getBlobEncryptionKey());
                }
                assertEquals("Expiration time mismatch in MessageInfo", Utils.Infinite_Time, resp.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0).getExpirationTimeInMs());
                releaseNettyBufUnderneathStream(stream);
            } catch (MessageFormatException e) {
                fail();
            }
        }
    }
    channel1.disconnect();
    channel2.disconnect();
    channel3.disconnect();
}