Examples with StoreKey com.github.ambry.store.StoreKey used on opensource projects

Example 26 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class AmbryRequests method handleGetRequest.

@Override
public void handleGetRequest(NetworkRequest request) throws IOException, InterruptedException {
    GetRequest getRequest;
    if (request instanceof LocalChannelRequest) {
        // This is a case where handleGetRequest is called when frontends are reading from Azure. In this case, this method
        // is called by request handler threads running within the frontend router itself. So, the request can be directly
        // referenced as java objects without any need for deserialization.
        getRequest = (GetRequest) ((LocalChannelRequest) request).getRequestInfo().getRequest();
    } else {
        getRequest = GetRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    }
    Histogram responseQueueTime = null;
    Histogram responseSendTime = null;
    Histogram responseTotalTime = null;
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    boolean isReplicaRequest = getRequest.getClientId().startsWith(GetRequest.Replication_Client_Id_Prefix);
    if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
        metrics.getBlobRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobRequestRate.mark();
        responseQueueTime = metrics.getBlobResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobSendTimeInMs;
        responseTotalTime = metrics.getBlobTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
        metrics.getBlobPropertiesRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobPropertiesRequestRate.mark();
        responseQueueTime = metrics.getBlobPropertiesResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobPropertiesSendTimeInMs;
        responseTotalTime = metrics.getBlobPropertiesTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
        metrics.getBlobUserMetadataRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobUserMetadataRequestRate.mark();
        responseQueueTime = metrics.getBlobUserMetadataResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobUserMetadataSendTimeInMs;
        responseTotalTime = metrics.getBlobUserMetadataTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
        metrics.getBlobInfoRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobInfoRequestRate.mark();
        responseQueueTime = metrics.getBlobInfoResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobInfoSendTimeInMs;
        responseTotalTime = metrics.getBlobInfoTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
        if (isReplicaRequest) {
            metrics.getBlobAllByReplicaRequestQueueTimeInMs.update(requestQueueTime);
            metrics.getBlobAllByReplicaRequestRate.mark();
            responseQueueTime = metrics.getBlobAllByReplicaResponseQueueTimeInMs;
            responseSendTime = metrics.getBlobAllByReplicaSendTimeInMs;
            responseTotalTime = metrics.getBlobAllByReplicaTotalTimeInMs;
        } else {
            metrics.getBlobAllRequestQueueTimeInMs.update(requestQueueTime);
            metrics.getBlobAllRequestRate.mark();
            responseQueueTime = metrics.getBlobAllResponseQueueTimeInMs;
            responseSendTime = metrics.getBlobAllSendTimeInMs;
            responseTotalTime = metrics.getBlobAllTotalTimeInMs;
        }
    }
    long startTime = SystemTime.getInstance().milliseconds();
    GetResponse response = null;
    try {
        List<Send> messagesToSendList = new ArrayList<>(getRequest.getPartitionInfoList().size());
        List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<>(getRequest.getPartitionInfoList().size());
        for (PartitionRequestInfo partitionRequestInfo : getRequest.getPartitionInfoList()) {
            ServerErrorCode error = validateRequest(partitionRequestInfo.getPartition(), RequestOrResponseType.GetRequest, false);
            if (error != ServerErrorCode.No_Error) {
                logger.error("Validating get request failed for partition {} with error {}", partitionRequestInfo.getPartition(), error);
                PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), error);
                partitionResponseInfoList.add(partitionResponseInfo);
            } else {
                try {
                    Store storeToGet = storeManager.getStore(partitionRequestInfo.getPartition());
                    EnumSet<StoreGetOptions> storeGetOptions = EnumSet.noneOf(StoreGetOptions.class);
                    // Currently only one option is supported.
                    if (getRequest.getGetOption() == GetOption.Include_Expired_Blobs) {
                        storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Expired);
                    }
                    if (getRequest.getGetOption() == GetOption.Include_Deleted_Blobs) {
                        storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted);
                    }
                    if (getRequest.getGetOption() == GetOption.Include_All) {
                        storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted, StoreGetOptions.Store_Include_Expired);
                    }
                    List<StoreKey> convertedStoreKeys = getConvertedStoreKeys(partitionRequestInfo.getBlobIds());
                    List<StoreKey> dedupedStoreKeys = convertedStoreKeys.size() > 1 ? convertedStoreKeys.stream().distinct().collect(Collectors.toList()) : convertedStoreKeys;
                    StoreInfo info = storeToGet.get(dedupedStoreKeys, storeGetOptions);
                    MessageFormatSend blobsToSend = new MessageFormatSend(info.getMessageReadSet(), getRequest.getMessageFormatFlag(), messageFormatMetrics, storeKeyFactory);
                    PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), info.getMessageReadSetInfo(), blobsToSend.getMessageMetadataList());
                    messagesToSendList.add(blobsToSend);
                    partitionResponseInfoList.add(partitionResponseInfo);
                } catch (StoreException e) {
                    boolean logInErrorLevel = false;
                    if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
                        metrics.idNotFoundError.inc();
                    } else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
                        metrics.ttlExpiredError.inc();
                    } else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
                        metrics.idDeletedError.inc();
                    } else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
                        metrics.getAuthorizationFailure.inc();
                    } else {
                        metrics.unExpectedStoreGetError.inc();
                        logInErrorLevel = true;
                    }
                    if (logInErrorLevel) {
                        logger.error("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
                    } else {
                        logger.trace("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
                    }
                    PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
                    partitionResponseInfoList.add(partitionResponseInfo);
                } catch (MessageFormatException e) {
                    logger.error("Message format exception on a get with error code {} for partitionRequestInfo {}", e.getErrorCode(), partitionRequestInfo, e);
                    if (e.getErrorCode() == MessageFormatErrorCodes.Data_Corrupt) {
                        metrics.dataCorruptError.inc();
                    } else if (e.getErrorCode() == MessageFormatErrorCodes.Unknown_Format_Version) {
                        metrics.unknownFormatError.inc();
                    }
                    PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getMessageFormatErrorMapping(e.getErrorCode()));
                    partitionResponseInfoList.add(partitionResponseInfo);
                }
            }
        }
        CompositeSend compositeSend = new CompositeSend(messagesToSendList);
        response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, compositeSend, ServerErrorCode.No_Error);
    } catch (Exception e) {
        logger.error("Unknown exception for request {}", getRequest, e);
        response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", getRequest, response, processingTime);
        long responseSize = response != null ? response.sizeInBytes() : 0;
        if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
            metrics.getBlobProcessingTimeInMs.update(processingTime);
            metrics.updateGetBlobProcessingTimeBySize(responseSize, processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
            metrics.getBlobPropertiesProcessingTimeInMs.update(processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
            metrics.getBlobUserMetadataProcessingTimeInMs.update(processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
            metrics.getBlobInfoProcessingTimeInMs.update(processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
            if (isReplicaRequest) {
                metrics.getBlobAllByReplicaProcessingTimeInMs.update(processingTime);
                // client id now has dc name at the end, for example: ClientId=replication-fetch-abc.example.com[dc1]
                String[] clientStrs = getRequest.getClientId().split("\\[");
                if (clientStrs.length > 1) {
                    String clientDc = clientStrs[1].substring(0, clientStrs[1].length() - 1);
                    if (!currentNode.getDatacenterName().equals(clientDc)) {
                        metrics.updateCrossColoFetchBytesRate(clientDc, responseSize);
                    }
                }
            } else {
                metrics.getBlobAllProcessingTimeInMs.update(processingTime);
                metrics.updateGetBlobProcessingTimeBySize(responseSize, processingTime);
            }
        }
    }
    sendGetResponse(requestResponseChannel, response, request, responseQueueTime, responseSendTime, responseTotalTime, totalTimeSpent, response.sizeInBytes(), getRequest.getMessageFormatFlag(), metrics);
}

Example 27 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class AmbryRequests method getConvertedStoreKeys.

/**
 * Convert StoreKeys based on {@link StoreKeyConverter}
 * @param storeKeys A list of original storeKeys.
 * @return A list of converted storeKeys.
 */
protected List<StoreKey> getConvertedStoreKeys(List<? extends StoreKey> storeKeys) throws Exception {
    storeKeyConverterFactory.getStoreKeyConverter().dropCache();
    Map<StoreKey, StoreKey> conversionMap = storeKeyConverterFactory.getStoreKeyConverter().convert(storeKeys);
    List<StoreKey> convertedStoreKeys = new ArrayList<>();
    for (StoreKey key : storeKeys) {
        StoreKey convertedKey = conversionMap.get(key);
        convertedStoreKeys.add(convertedKey == null ? key : convertedKey);
    }
    return convertedStoreKeys;
}

Example 28 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class AmbryRequests method handleUndeleteRequest.

@Override
public void handleUndeleteRequest(NetworkRequest request) throws IOException, InterruptedException {
    UndeleteRequest undeleteRequest;
    if (request instanceof LocalChannelRequest) {
        // This is a case where handleUndeleteRequest is called when frontends are talking to Azure. In this case, this method
        // is called by request handler threads running within the frontend router itself. So, the request can be directly
        // referenced as java objects without any need for deserialization.
        undeleteRequest = (UndeleteRequest) ((LocalChannelRequest) request).getRequestInfo().getRequest();
    } else {
        undeleteRequest = UndeleteRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    }
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.undeleteBlobRequestQueueTimeInMs.update(requestQueueTime);
    metrics.undeleteBlobRequestRate.mark();
    long startTime = SystemTime.getInstance().milliseconds();
    UndeleteResponse response = null;
    Store storeToUndelete;
    StoreKey convertedStoreKey;
    try {
        convertedStoreKey = getConvertedStoreKeys(Collections.singletonList(undeleteRequest.getBlobId())).get(0);
        ServerErrorCode error = validateRequest(undeleteRequest.getBlobId().getPartition(), RequestOrResponseType.UndeleteRequest, false);
        if (error != ServerErrorCode.No_Error) {
            logger.error("Validating undelete request failed with error {} for request {}", error, undeleteRequest);
            response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), error);
        } else {
            BlobId convertedBlobId = (BlobId) convertedStoreKey;
            MessageInfo info = new MessageInfo.Builder(convertedBlobId, -1, convertedBlobId.getAccountId(), convertedBlobId.getContainerId(), undeleteRequest.getOperationTimeMs()).isUndeleted(true).lifeVersion(MessageInfo.LIFE_VERSION_FROM_FRONTEND).build();
            storeToUndelete = storeManager.getStore(undeleteRequest.getBlobId().getPartition());
            short lifeVersion = storeToUndelete.undelete(info);
            response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), lifeVersion);
            if (notification != null) {
                notification.onBlobReplicaUndeleted(currentNode.getHostname(), currentNode.getPort(), convertedStoreKey.getID(), BlobReplicaSourceType.PRIMARY);
            }
        }
    } catch (StoreException e) {
        boolean logInErrorLevel = false;
        if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
            metrics.idNotFoundError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
            metrics.ttlExpiredError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted_Permanently) {
            metrics.idDeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.Life_Version_Conflict) {
            metrics.lifeVersionConflictError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Not_Deleted) {
            metrics.idNotDeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Undeleted) {
            metrics.idUndeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
            metrics.undeleteAuthorizationFailure.inc();
        } else {
            logInErrorLevel = true;
            metrics.unExpectedStoreUndeleteError.inc();
        }
        if (logInErrorLevel) {
            logger.error("Store exception on a undelete with error code {} for request {}", e.getErrorCode(), undeleteRequest, e);
        } else {
            logger.trace("Store exception on a undelete with error code {} for request {}", e.getErrorCode(), undeleteRequest, e);
        }
        if (e.getErrorCode() == StoreErrorCodes.ID_Undeleted) {
            if (e instanceof IdUndeletedStoreException) {
                response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), ((IdUndeletedStoreException) e).getLifeVersion(), ServerErrorCode.Blob_Already_Undeleted);
            } else {
                response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), MessageInfo.LIFE_VERSION_FROM_FRONTEND, ServerErrorCode.Blob_Already_Undeleted);
            }
        } else {
            response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
        }
    } catch (Exception e) {
        logger.error("Unknown exception for undelete request {}", undeleteRequest, e);
        response = new UndeleteResponse(undeleteRequest.getCorrelationId(), undeleteRequest.getClientId(), ServerErrorCode.Unknown_Error);
        metrics.unExpectedStoreUndeleteError.inc();
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", undeleteRequest, response, processingTime);
        metrics.undeleteBlobProcessingTimeInMs.update(processingTime);
    }
    requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.undeleteBlobResponseQueueTimeInMs, metrics.undeleteBlobSendTimeInMs, metrics.undeleteBlobTotalTimeInMs, null, null, totalTimeSpent));
}

Example 29 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class MessageInfoAndMetadataListSerDeTest method testSerDe.

@Test
public void testSerDe() throws Exception {
    MockClusterMap mockMap = new MockClusterMap();
    MockPartitionId partitionId = new MockPartitionId();
    short[] accountIds = { 100, 101, 102, 103 };
    short[] containerIds = { 10, 11, 12, 13 };
    boolean[] isDeletedVals = { false, true, false, true };
    boolean[] isTtlUpdatedVals = { true, false, false, true };
    boolean[] isUndeletedVals = { true, false, false, true };
    short[] lifeVersionVals = { 1, 2, 3, 4 };
    Long[] crcs = { null, 100L, Long.MIN_VALUE, Long.MAX_VALUE };
    StoreKey[] keys = { new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[0], containerIds[0], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[1], containerIds[1], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[2], containerIds[2], partitionId, false, BlobId.BlobDataType.DATACHUNK), new BlobId(TestUtils.getRandomElement(BlobId.getAllValidVersions()), BlobId.BlobIdType.NATIVE, (byte) 0, accountIds[3], containerIds[3], partitionId, false, BlobId.BlobDataType.DATACHUNK) };
    long[] blobSizes = { 1024, 2048, 4096, 8192 };
    long[] times = { SystemTime.getInstance().milliseconds(), SystemTime.getInstance().milliseconds() - 1, SystemTime.getInstance().milliseconds() + TimeUnit.SECONDS.toMillis(5), Utils.Infinite_Time };
    MessageMetadata[] messageMetadata = new MessageMetadata[4];
    messageMetadata[0] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(100)));
    messageMetadata[1] = new MessageMetadata(null);
    messageMetadata[2] = null;
    messageMetadata[3] = new MessageMetadata(ByteBuffer.wrap(getRandomBytes(200)));
    List<MessageInfo> messageInfoList = new ArrayList<>(4);
    List<MessageMetadata> messageMetadataList = new ArrayList<>(4);
    for (int i = 0; i < 4; i++) {
        messageInfoList.add(new MessageInfo(keys[i], blobSizes[i], isDeletedVals[i], isTtlUpdatedVals[i], isUndeletedVals[i], times[i], crcs[i], accountIds[i], containerIds[i], times[i], lifeVersionVals[i]));
        messageMetadataList.add(messageMetadata[i]);
    }
    // Serialize and then deserialize
    MessageInfoAndMetadataListSerde messageInfoAndMetadataListSerde = new MessageInfoAndMetadataListSerde(messageInfoList, messageMetadataList, serDeVersion);
    ByteBuf buffer = Unpooled.buffer(messageInfoAndMetadataListSerde.getMessageInfoAndMetadataListSize());
    messageInfoAndMetadataListSerde.serializeMessageInfoAndMetadataList(buffer);
    if (serDeVersion >= MessageInfoAndMetadataListSerde.VERSION_5) {
        // should fail if the wrong version is provided
        try {
            MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, (short) (serDeVersion - 1));
            Assert.fail("Should have failed to deserialize");
        } catch (IllegalArgumentException e) {
        // expected. Nothing to do
        }
    }
    buffer.readerIndex(0);
    MessageInfoAndMetadataListSerde messageInfoAndMetadataList = MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(new NettyByteBufDataInputStream(buffer), mockMap, serDeVersion);
    // Verify
    List<MessageInfo> responseMessageInfoList = messageInfoAndMetadataList.getMessageInfoList();
    List<MessageMetadata> responseMessageMetadataList = messageInfoAndMetadataList.getMessageMetadataList();
    Assert.assertEquals(4, responseMessageInfoList.size());
    Assert.assertEquals(4, responseMessageMetadataList.size());
    for (int i = 0; i < 4; i++) {
        assertMessageInfoEquality(messageInfoList.get(i), responseMessageInfoList.get(i));
        assertMessageMetadataEquality(messageMetadataList.get(i), responseMessageMetadataList.get(i));
    }
}

Example 30 with StoreKey

use of com.github.ambry.store.StoreKey in project ambry by linkedin.

the class ReplicaThread method getMessagesForMissingKeys.

/**
 * Gets the messages for the keys that are missing from the local store by issuing a {@link GetRequest} to the remote
 * node, if there are any missing keys. If there are no missing keys to be fetched, then no request is issued and a
 * null response is returned.
 * @param connectedChannel The connection channel to the remote node
 * @param exchangeMetadataResponseList The list of metadata response from the remote node
 * @param replicasToReplicatePerNode The list of remote replicas for the remote node
 * @param remoteNode The remote node from which replication needs to happen
 * @param remoteColoGetRequestForStandby boolean which indicates if we are getting missing keys for standby or
 *                                       non-leader replica pairs during leader-based replication.
 * @return The response that contains the missing messages; or null if no request was issued because there were no
 * keys missing.
 * @throws ReplicationException
 * @throws IOException
 */
private GetResponse getMessagesForMissingKeys(ConnectedChannel connectedChannel, List<ExchangeMetadataResponse> exchangeMetadataResponseList, List<RemoteReplicaInfo> replicasToReplicatePerNode, DataNodeId remoteNode, boolean remoteColoGetRequestForStandby) throws ReplicationException, IOException {
    List<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    for (int i = 0; i < exchangeMetadataResponseList.size(); i++) {
        ExchangeMetadataResponse exchangeMetadataResponse = exchangeMetadataResponseList.get(i);
        RemoteReplicaInfo remoteReplicaInfo = replicasToReplicatePerNode.get(i);
        if (exchangeMetadataResponse.serverErrorCode == ServerErrorCode.No_Error) {
            Set<StoreKey> missingStoreKeys = exchangeMetadataResponse.getMissingStoreKeys();
            if (missingStoreKeys.size() > 0) {
                if (remoteNode instanceof CloudDataNode) {
                    logger.trace("Replicating blobs from CloudDataNode: {}", missingStoreKeys);
                }
                ArrayList<BlobId> keysToFetch = new ArrayList<BlobId>();
                for (StoreKey storeKey : missingStoreKeys) {
                    keysToFetch.add((BlobId) storeKey);
                }
                PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(remoteReplicaInfo.getReplicaId().getPartitionId(), keysToFetch);
                partitionRequestInfoList.add(partitionRequestInfo);
            }
        }
    }
    GetResponse getResponse = null;
    if (!partitionRequestInfoList.isEmpty()) {
        GetRequest getRequest = new GetRequest(correlationIdGenerator.incrementAndGet(), GetRequest.Replication_Client_Id_Prefix + dataNodeId.getHostname() + "[" + dataNodeId.getDatacenterName() + "]", MessageFormatFlags.All, partitionRequestInfoList, replicationConfig.replicationIncludeAll ? GetOption.Include_All : GetOption.None);
        long startTime = time.milliseconds();
        try {
            ChannelOutput channelOutput = connectedChannel.sendAndReceive(getRequest);
            getResponse = GetResponse.readFrom(channelOutput.getInputStream(), clusterMap);
            long getRequestTime = time.milliseconds() - startTime;
            replicationMetrics.updateGetRequestTime(getRequestTime, replicatingFromRemoteColo, replicatingOverSsl, datacenterName, remoteColoGetRequestForStandby);
            if (getResponse.getError() != ServerErrorCode.No_Error) {
                logger.error("Remote node: {} Thread name: {} Remote replicas: {} GetResponse from replication: {}", remoteNode, threadName, replicasToReplicatePerNode, getResponse.getError());
                throw new ReplicationException(" Get Request returned error when trying to get missing keys " + getResponse.getError());
            }
        } catch (IOException e) {
            responseHandler.onEvent(replicasToReplicatePerNode.get(0).getReplicaId(), e);
            throw e;
        }
    }
    return getResponse;
}