Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 26 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method applyUpdatesToBlobInLocalStore.

/**
 * Compares blob metadata in received message from remote replica with the blob in local store and updates its
 * ttl_update/delete/undelete properties. This is called when a replicated blob from remote replica is found on local store.
 * @param messageInfo message information of the blob from remote replica
 * @param remoteReplicaInfo remote replica information
 * @param localKey local blob information
 * @throws StoreException
 */
public void applyUpdatesToBlobInLocalStore(MessageInfo messageInfo, RemoteReplicaInfo remoteReplicaInfo, BlobId localKey) throws StoreException {
    MessageInfo localMessageInfo = remoteReplicaInfo.getLocalStore().findKey(localKey);
    boolean deletedLocally = localMessageInfo.isDeleted();
    boolean ttlUpdatedLocally = localMessageInfo.isTtlUpdated();
    short localLifeVersion = localMessageInfo.getLifeVersion();
    short remoteLifeVersion = messageInfo.getLifeVersion();
    if (localLifeVersion > remoteLifeVersion) {
    // if the lifeVersion in local store is greater than the remote lifeVersion, then nothing needs to be done.
    } else if (localLifeVersion == remoteLifeVersion) {
        // we are operating in the same version, in this case, delete would be the final state.
        if (!deletedLocally) {
            // Only adds record when it's not deleted yet. Since delete is the final state for this lifeVersion, if there
            // is a delete record for the current lifeVersion, then nothing needs to be done.
            MessageInfo info = new MessageInfo(localKey, 0, localKey.getAccountId(), localKey.getContainerId(), messageInfo.getOperationTimeMs(), remoteLifeVersion);
            if (messageInfo.isTtlUpdated() && !ttlUpdatedLocally) {
                applyTtlUpdate(info, remoteReplicaInfo);
            }
            if (messageInfo.isDeleted()) {
                applyDelete(info, remoteReplicaInfo);
            }
        }
    } else {
        // if we are here, then the remote lifeVersion is greater than the local lifeVersion.
        // we need to reconcile the local state with the remote state.
        // 
        // There are three states we have to reconcile: lifeVersion, ttl_update, is_deleted.
        // To reconcile lifeVersion and is_deleted, we have to add a Delete or Undelete record, based on what the final state is.
        // to reconcile ttl_update, if the final state is delete, then, we have to add ttl_update before delete, other, we can add ttl_update after undelete.
        MessageInfo info = new MessageInfo(localKey, 0, localKey.getAccountId(), localKey.getContainerId(), messageInfo.getOperationTimeMs(), remoteLifeVersion);
        boolean shouldInsertTtlUpdate = false;
        if (messageInfo.isTtlUpdated() && !ttlUpdatedLocally) {
            // if the remote state is delete, then we can't insert TTL_UPDATE after delete, we have to insert a ttl_update here
            if (messageInfo.isDeleted()) {
                // this lifeVersion.
                if (deletedLocally) {
                    applyUndelete(info, remoteReplicaInfo);
                }
                applyTtlUpdate(info, remoteReplicaInfo);
            } else {
                // if final state is not delete, then to bump lifeVersion in local store to remote lifeVersion, we have to
                // add a undelete, and then add a ttl update.
                shouldInsertTtlUpdate = true;
            }
        }
        // if we are here, then the ttl update is matched
        if (messageInfo.isDeleted()) {
            applyDelete(info, remoteReplicaInfo);
        } else {
            applyUndelete(info, remoteReplicaInfo);
            if (shouldInsertTtlUpdate) {
                applyTtlUpdate(info, remoteReplicaInfo);
            }
        }
    }
}

Example 27 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method writeMessagesToLocalStoreAndAdvanceTokens.

/**
 * Writes the messages (if any) to the local stores from the remote stores for the missing keys, and advances tokens.
 * @param exchangeMetadataResponseList The list of metadata response from the remote node
 * @param getResponse The {@link GetResponse} that contains the missing messages. This may be null if there are no
 *                    missing messages to write as per the exchange metadata response. In that case this method will
 *                    simply advance the tokens for every store.
 * @param replicasToReplicatePerNode The list of remote replicas for the remote node
 * @param remoteNode The remote node from which replication needs to happen
 * @param remoteColoGetRequestForStandby boolean which indicates if we are getting missing keys for standby or
 *                                       non-leader replica pairs during leader-based replication.
 * @throws IOException
 */
private void writeMessagesToLocalStoreAndAdvanceTokens(List<ExchangeMetadataResponse> exchangeMetadataResponseList, GetResponse getResponse, List<RemoteReplicaInfo> replicasToReplicatePerNode, DataNodeId remoteNode, boolean remoteColoGetRequestForStandby) throws IOException {
    int partitionResponseInfoIndex = 0;
    long totalBytesFixed = 0;
    long totalBlobsFixed = 0;
    long startTime = time.milliseconds();
    for (int i = 0; i < exchangeMetadataResponseList.size(); i++) {
        ExchangeMetadataResponse exchangeMetadataResponse = exchangeMetadataResponseList.get(i);
        RemoteReplicaInfo remoteReplicaInfo = replicasToReplicatePerNode.get(i);
        // TODO: if remoteReplicaInfo.getLocalStore() is closed, write will fail
        if (exchangeMetadataResponse.serverErrorCode == ServerErrorCode.No_Error) {
            if (exchangeMetadataResponse.missingStoreMessages.size() > 0) {
                PartitionResponseInfo partitionResponseInfo = getResponse.getPartitionResponseInfoList().get(partitionResponseInfoIndex);
                responseHandler.onEvent(remoteReplicaInfo.getReplicaId(), partitionResponseInfo.getErrorCode());
                partitionResponseInfoIndex++;
                if (!partitionResponseInfo.getPartition().toPathString().equals(remoteReplicaInfo.getReplicaId().getPartitionId().toPathString())) {
                    throw new IllegalStateException("The partition id from partitionResponseInfo " + partitionResponseInfo.getPartition() + " and from remoteReplicaInfo " + remoteReplicaInfo.getReplicaId().getPartitionId() + " are not the same");
                }
                if (partitionResponseInfo.getErrorCode() == ServerErrorCode.No_Error) {
                    List<MessageInfo> messageInfoList = partitionResponseInfo.getMessageInfoList();
                    try {
                        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Messages to fix: {} " + "Partition: {} Local mount path: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.getMissingStoreKeys(), remoteReplicaInfo.getReplicaId().getPartitionId(), remoteReplicaInfo.getLocalReplicaId().getMountPath());
                        MessageFormatWriteSet writeset;
                        MessageSievingInputStream validMessageDetectionInputStream = new MessageSievingInputStream(getResponse.getInputStream(), messageInfoList, Collections.singletonList(transformer), metricRegistry);
                        if (validMessageDetectionInputStream.hasInvalidMessages()) {
                            replicationMetrics.incrementInvalidMessageError(partitionResponseInfo.getPartition());
                            logger.error("Out of {} messages, {} invalid messages were found in message stream from {}", messageInfoList.size(), messageInfoList.size() - validMessageDetectionInputStream.getValidMessageInfoList().size(), remoteReplicaInfo.getReplicaId());
                        }
                        messageInfoList = validMessageDetectionInputStream.getValidMessageInfoList();
                        if (messageInfoList.size() == 0) {
                            logger.debug("MessageInfoList is of size 0 as all messages are invalidated, deprecated, deleted or expired.");
                        } else {
                            writeset = new MessageFormatWriteSet(validMessageDetectionInputStream, messageInfoList, false);
                            remoteReplicaInfo.getLocalStore().put(writeset);
                        }
                        for (MessageInfo messageInfo : messageInfoList) {
                            totalBytesFixed += messageInfo.getSize();
                            logger.trace("Remote node: {} Thread name: {} Remote replica: {} Message replicated: {} Partition: {} " + "Local mount path: {} Message size: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey(), remoteReplicaInfo.getReplicaId().getPartitionId(), remoteReplicaInfo.getLocalReplicaId().getMountPath(), messageInfo.getSize());
                            if (notification != null) {
                                notification.onBlobReplicaCreated(dataNodeId.getHostname(), dataNodeId.getPort(), messageInfo.getStoreKey().getID(), BlobReplicaSourceType.REPAIRED);
                            }
                            if (messageInfo.isTtlUpdated()) {
                                applyTtlUpdate(messageInfo, remoteReplicaInfo);
                            }
                        }
                        totalBlobsFixed += messageInfoList.size();
                        if (leaderBasedReplicationAdmin != null) {
                            // If leader based replication is enabled, we will only fetch missing blobs for local leaders from their
                            // remote leaders. For non-leader replicas pairs (leader <-> standby, standby <-> leader, standby <->
                            // standby), we will store the missing keys and track them via leader<->leader exchanges and intra-dc
                            // replication.
                            // Notify all the replicas of the partition on newly written messages so that non-leader replica pairs
                            // can update their missing keys and advance token if needed.
                            leaderBasedReplicationAdmin.onMessageWriteForPartition(partitionResponseInfo.getPartition(), messageInfoList);
                        }
                        remoteReplicaInfo.setToken(exchangeMetadataResponse.remoteToken);
                        remoteReplicaInfo.setLocalLagFromRemoteInBytes(exchangeMetadataResponse.localLagFromRemoteInBytes);
                        // reset stored metadata response for this replica
                        remoteReplicaInfo.setExchangeMetadataResponse(new ExchangeMetadataResponse(ServerErrorCode.No_Error));
                        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token after speaking to remote node: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.remoteToken);
                    } catch (StoreException e) {
                        if (e.getErrorCode() != StoreErrorCodes.Already_Exist) {
                            replicationMetrics.updateLocalStoreError(remoteReplicaInfo.getReplicaId());
                            logger.error("Remote node: {} Thread name: {} Remote replica: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), e);
                        }
                    }
                } else if (partitionResponseInfo.getErrorCode() == ServerErrorCode.Blob_Deleted) {
                    replicationMetrics.blobDeletedOnGetCount.inc();
                    logger.trace("One of the blobs to GET is deleted: Remote node: {} Thread name: {} Remote replica: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId());
                } else if (partitionResponseInfo.getErrorCode() == ServerErrorCode.Blob_Authorization_Failure) {
                    replicationMetrics.blobAuthorizationFailureCount.inc();
                    logger.error("One of the blobs authorization failed: Remote node: {} Thread name: {} Remote replica: {} Keys are: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.getMissingStoreKeys());
                } else {
                    replicationMetrics.updateGetRequestError(remoteReplicaInfo.getReplicaId());
                    logger.error("Remote node: {} Thread name: {} Remote replica: {} Server error: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), partitionResponseInfo.getErrorCode());
                }
            }
        }
    }
    long batchStoreWriteTime = time.milliseconds() - startTime;
    replicationMetrics.updateBatchStoreWriteTime(batchStoreWriteTime, totalBytesFixed, totalBlobsFixed, replicatingFromRemoteColo, replicatingOverSsl, datacenterName, remoteColoGetRequestForStandby);
}

Example 28 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method processMissingKeysFromPreviousMetadataResponse.

/**
 * Compare message infos of remote standby replica (whose blobs are now received from leader replicas) with message info
 * of blobs in local store and reconcile blob properties like ttl_update, delete, undelete. If blobs for all the missing messages
 * of the standby replica are received and updated, move the remote token of the standby forward.
 * @param remoteReplicaInfo remote replica information
 */
void processMissingKeysFromPreviousMetadataResponse(RemoteReplicaInfo remoteReplicaInfo) {
    try {
        ExchangeMetadataResponse exchangeMetadataResponse = remoteReplicaInfo.getExchangeMetadataResponse();
        if (!exchangeMetadataResponse.isEmpty()) {
            Set<MessageInfo> receivedStoreMessagesWithUpdatesPending = exchangeMetadataResponse.getReceivedStoreMessagesWithUpdatesPending();
            Set<MessageInfo> receivedMessagesWithUpdatesCompleted = new HashSet<>();
            // updates to them (if needed) to reconcile delete, ttl_update and undelete states.
            for (MessageInfo messageInfo : receivedStoreMessagesWithUpdatesPending) {
                BlobId localStoreKey = (BlobId) exchangeMetadataResponse.remoteKeyToLocalKeyMap.get(messageInfo.getStoreKey());
                if (localStoreKey != null) {
                    applyUpdatesToBlobInLocalStore(messageInfo, remoteReplicaInfo, localStoreKey);
                }
                receivedMessagesWithUpdatesCompleted.add(messageInfo);
            }
            // 2. Remove the messages whose updates have been completed
            exchangeMetadataResponse.removeReceivedStoreMessagesWithUpdatesPending(receivedMessagesWithUpdatesCompleted);
            // updates for them have been completed, move the remote token forward and update local lag from remote for this replica.
            if (exchangeMetadataResponse.isEmpty()) {
                remoteReplicaInfo.setToken(exchangeMetadataResponse.remoteToken);
                remoteReplicaInfo.setLocalLagFromRemoteInBytes(exchangeMetadataResponse.localLagFromRemoteInBytes);
                logger.trace("Thread name: {}, updating token {} and lag {} for partition {} for Remote replica {}", threadName, exchangeMetadataResponse.remoteToken, exchangeMetadataResponse.localLagFromRemoteInBytes, remoteReplicaInfo.getReplicaId().getPartitionId().toString(), remoteReplicaInfo.getReplicaId());
                remoteReplicaInfo.setExchangeMetadataResponse(new ExchangeMetadataResponse(ServerErrorCode.No_Error));
            }
        }
    } catch (StoreException e) {
        if (e.getErrorCode() == StoreErrorCodes.Store_Not_Started) {
            logger.info("Local store not started for remote replica: {}", remoteReplicaInfo.getReplicaId());
        } else {
            logger.error("Exception occurred while updating blob from Remote replica {} for partition: {}", remoteReplicaInfo.getReplicaId(), remoteReplicaInfo.getReplicaId().getPartitionId().toString(), e);
            replicationMetrics.updateLocalStoreError(remoteReplicaInfo.getReplicaId());
        }
        // reset stored metadata response so that metadata request is sent again for this replica
        remoteReplicaInfo.setExchangeMetadataResponse(new ExchangeMetadataResponse(ServerErrorCode.No_Error));
    }
}

Example 29 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method getMissingStoreMessages.

/**
 * Gets the missing store messages by comparing the messages from the remote node
 * @param replicaMetadataResponseInfo The response that contains the messages from the remote node
 * @param remoteNode The remote node from which replication needs to happen
 * @param remoteReplicaInfo The remote replica that contains information about the remote replica id
 * @return List of store messages that are missing from the local store
 * @throws StoreException if store error (usually IOError) occurs when getting missing keys.
 */
Set<MessageInfo> getMissingStoreMessages(ReplicaMetadataResponseInfo replicaMetadataResponseInfo, DataNodeId remoteNode, RemoteReplicaInfo remoteReplicaInfo) throws StoreException {
    long startTime = time.milliseconds();
    List<MessageInfo> messageInfoList = replicaMetadataResponseInfo.getMessageInfoList();
    Map<MessageInfo, StoreKey> remoteMessageToConvertedKeyNonNull = new HashMap<>();
    for (MessageInfo messageInfo : messageInfoList) {
        StoreKey storeKey = messageInfo.getStoreKey();
        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key from remote: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), storeKey);
        StoreKey convertedKey = storeKeyConverter.getConverted(storeKey);
        if (skipPredicate == null) {
            logger.debug("SkipPredicate is null");
        }
        if (convertedKey != null && (!replicationConfig.replicationContainerDeletionEnabled || skipPredicate == null || !skipPredicate.test(messageInfo))) {
            remoteMessageToConvertedKeyNonNull.put(messageInfo, convertedKey);
        }
    }
    Set<StoreKey> convertedMissingStoreKeys = remoteReplicaInfo.getLocalStore().findMissingKeys(new ArrayList<>(remoteMessageToConvertedKeyNonNull.values()));
    Set<MessageInfo> missingRemoteMessages = new HashSet<>();
    remoteMessageToConvertedKeyNonNull.forEach((messageInfo, convertedKey) -> {
        if (convertedMissingStoreKeys.contains(convertedKey)) {
            logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key missing id (converted): {} Key missing id (remote): {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), convertedKey, messageInfo.getStoreKey());
            missingRemoteMessages.add(messageInfo);
        }
    });
    if (messageInfoList.size() != 0 && missingRemoteMessages.size() == 0) {
        // Catching up
        replicationMetrics.allResponsedKeysExist.inc();
    }
    replicationMetrics.updateCheckMissingKeysTime(time.milliseconds() - startTime, replicatingFromRemoteColo, datacenterName);
    return missingRemoteMessages;
}

Example 30 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method exchangeMetadata.

/**
 * Gets all the metadata about messages from the remote replicas since last token. Checks the messages with the local
 * store and finds all the messages that are missing. For the messages that are not missing, updates the delete
 * and ttl state.
 * @param connectedChannel The connected channel that represents a connection to the remote replica
 * @param replicasToReplicatePerNode The information about the replicas that is being replicated
 * @return - List of ExchangeMetadataResponse that contains the set of store keys that are missing from the local
 *           store and are present in the remote replicas and also the new token from the remote replicas
 * @throws IOException
 * @throws ReplicationException
 */
List<ExchangeMetadataResponse> exchangeMetadata(ConnectedChannel connectedChannel, List<RemoteReplicaInfo> replicasToReplicatePerNode) throws IOException, ReplicationException {
    long exchangeMetadataStartTimeInMs = time.milliseconds();
    List<ExchangeMetadataResponse> exchangeMetadataResponseList = new ArrayList<>();
    if (replicasToReplicatePerNode.size() > 0) {
        try {
            DataNodeId remoteNode = replicasToReplicatePerNode.get(0).getReplicaId().getDataNodeId();
            ReplicaMetadataResponse response = getReplicaMetadataResponse(replicasToReplicatePerNode, connectedChannel, remoteNode);
            long startTimeInMs = time.milliseconds();
            Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = batchConvertReplicaMetadataResponseKeys(response);
            for (int i = 0; i < response.getReplicaMetadataResponseInfoList().size(); i++) {
                RemoteReplicaInfo remoteReplicaInfo = replicasToReplicatePerNode.get(i);
                ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
                responseHandler.onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
                if (replicaMetadataResponseInfo.getError() == ServerErrorCode.No_Error) {
                    // Skip stores that were stopped during call to getReplicaMetadataResponse
                    if (!remoteReplicaInfo.getLocalStore().isStarted()) {
                        exchangeMetadataResponseList.add(new ExchangeMetadataResponse(ServerErrorCode.Temporarily_Disabled));
                    } else {
                        try {
                            logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token from remote: {} Replica lag: {} ", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getFindToken(), replicaMetadataResponseInfo.getRemoteReplicaLagInBytes());
                            Set<MessageInfo> remoteMissingStoreMessages = getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
                            processReplicaMetadataResponse(remoteMissingStoreMessages, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
                            // Get the converted keys for the missing keys of this replica (to store them along with missing keys in
                            // the exchange metadata response). For leader based replication, these are used during processing
                            // of missing keys for non-leader replica pairs which will come later via leader<->leader replication.
                            Map<StoreKey, StoreKey> remoteKeyToLocalKeySubMap = new HashMap<>();
                            remoteMissingStoreMessages.forEach(remoteMissingStoreMessage -> {
                                StoreKey remoteKey = remoteMissingStoreMessage.getStoreKey();
                                remoteKeyToLocalKeySubMap.put(remoteKey, remoteKeyToLocalKeyMap.get(remoteKey));
                            });
                            ExchangeMetadataResponse exchangeMetadataResponse = new ExchangeMetadataResponse(remoteMissingStoreMessages, replicaMetadataResponseInfo.getFindToken(), replicaMetadataResponseInfo.getRemoteReplicaLagInBytes(), remoteKeyToLocalKeySubMap, time);
                            // update replication lag in ReplicaSyncUpManager
                            if (replicaSyncUpManager != null && remoteReplicaInfo.getLocalStore().getCurrentState() == ReplicaState.BOOTSTRAP) {
                                ReplicaId localReplica = remoteReplicaInfo.getLocalReplicaId();
                                ReplicaId remoteReplica = remoteReplicaInfo.getReplicaId();
                                boolean isSyncCompleted = replicaSyncUpManager.updateReplicaLagAndCheckSyncStatus(localReplica, remoteReplica, exchangeMetadataResponse.localLagFromRemoteInBytes, ReplicaState.STANDBY);
                                // if catchup is completed by this update call, we can complete bootstrap in local store
                                if (isSyncCompleted) {
                                    // complete BOOTSTRAP -> STANDBY transition
                                    remoteReplicaInfo.getLocalStore().setCurrentState(ReplicaState.STANDBY);
                                    remoteReplicaInfo.getLocalStore().completeBootstrap();
                                }
                            }
                            // If remote token has not moved forward, wait for back off time before resending next metadata request
                            if (remoteReplicaInfo.getToken().equals(exchangeMetadataResponse.remoteToken)) {
                                remoteReplicaInfo.setReEnableReplicationTime(time.milliseconds() + replicationConfig.replicationSyncedReplicaBackoffDurationMs);
                                syncedBackOffCount.inc();
                            }
                            // There are no missing keys. We just advance the token
                            if (exchangeMetadataResponse.missingStoreMessages.size() == 0) {
                                remoteReplicaInfo.setToken(exchangeMetadataResponse.remoteToken);
                                remoteReplicaInfo.setLocalLagFromRemoteInBytes(exchangeMetadataResponse.localLagFromRemoteInBytes);
                                logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token after speaking to remote node: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.remoteToken);
                            }
                            replicationMetrics.updateLagMetricForRemoteReplica(remoteReplicaInfo, exchangeMetadataResponse.localLagFromRemoteInBytes);
                            if (replicaMetadataResponseInfo.getMessageInfoList().size() > 0) {
                                replicationMetrics.updateCatchupPointMetricForCloudReplica(remoteReplicaInfo, replicaMetadataResponseInfo.getMessageInfoList().get(replicaMetadataResponseInfo.getMessageInfoList().size() - 1).getOperationTimeMs());
                            }
                            // Add exchangeMetadataResponse to list at the end after operations such as replicaSyncUpManager(if not null)
                            // has completed update, etc. The reason is we may get exceptions in between (for ex: replicaSyncUpManager may
                            // throw exception) and end up adding one more exchangeMetadataResponse associated with same RemoteReplicaInfo.
                            exchangeMetadataResponseList.add(exchangeMetadataResponse);
                        } catch (Exception e) {
                            if (e instanceof StoreException && ((StoreException) e).getErrorCode() == StoreErrorCodes.Store_Not_Started) {
                                // Must have just been stopped, just skip it and move on.
                                logger.info("Local store not started for remote replica: {}", remoteReplicaInfo.getReplicaId());
                                exchangeMetadataResponseList.add(new ExchangeMetadataResponse(ServerErrorCode.Temporarily_Disabled));
                            } else {
                                logger.error("Remote node: {} Thread name: {} Remote replica: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), e);
                                replicationMetrics.updateLocalStoreError(remoteReplicaInfo.getReplicaId());
                                responseHandler.onEvent(remoteReplicaInfo.getReplicaId(), e);
                                exchangeMetadataResponseList.add(new ExchangeMetadataResponse(ServerErrorCode.Unknown_Error));
                            }
                        }
                    }
                } else {
                    replicationMetrics.updateMetadataRequestError(remoteReplicaInfo.getReplicaId());
                    logger.error("Remote node: {} Thread name: {} Remote replica: {} Server error: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
                    exchangeMetadataResponseList.add(new ExchangeMetadataResponse(replicaMetadataResponseInfo.getError()));
                }
                if (replicatingFromRemoteColo && leaderBasedReplicationAdmin != null) {
                    ExchangeMetadataResponse exchangeMetadataResponse = exchangeMetadataResponseList.get(i);
                    if (exchangeMetadataResponse.serverErrorCode.equals(ServerErrorCode.No_Error)) {
                        // If leader-based replication is enabled, store the meta data exchange received for the remote replica as
                        // standby replicas will not send GET request for the missing store keys and track them from leader <->
                        // leader exchanges and intra-dc replication.
                        remoteReplicaInfo.setExchangeMetadataResponse(new ExchangeMetadataResponse(exchangeMetadataResponse));
                        // It is possible that some of the missing keys found in exchange metadata response are written in parallel
                        // by other replica threads since the time we calculated it. Go through the local store once more and
                        // update missing keys set stored in the exchangeMetadataResponse for the remote replica.
                        refreshMissingStoreMessagesForStandbyReplica(remoteReplicaInfo);
                    }
                }
            }
            long processMetadataResponseTimeInMs = time.milliseconds() - startTimeInMs;
            logger.trace("Remote node: {} Thread name: {} processMetadataResponseTime: {}", remoteNode, threadName, processMetadataResponseTimeInMs);
        } finally {
            long exchangeMetadataTime = time.milliseconds() - exchangeMetadataStartTimeInMs;
            replicationMetrics.updateExchangeMetadataTime(exchangeMetadataTime, replicatingFromRemoteColo, replicatingOverSsl, datacenterName);
        }
    }
    return exchangeMetadataResponseList;
}