Examples with MessageInfo com.github.ambry.store.MessageInfo used on opensource projects

Example 61 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStore method get.

@Override
public StoreInfo get(List<? extends StoreKey> ids, EnumSet<StoreGetOptions> storeGetOptions) throws StoreException {
    checkStarted();
    checkStoreKeyDuplicates(ids);
    List<CloudMessageReadSet.BlobReadInfo> blobReadInfos = new ArrayList<>(ids.size());
    List<MessageInfo> messageInfos = new ArrayList<>(ids.size());
    try {
        List<BlobId> blobIdList = ids.stream().map(key -> (BlobId) key).collect(Collectors.toList());
        Map<String, CloudBlobMetadata> cloudBlobMetadataListMap = requestAgent.doWithRetries(() -> cloudDestination.getBlobMetadata(blobIdList), "GetBlobMetadata", partitionId.toPathString());
        // Throw StoreException with ID_Not_Found if cloudBlobMetadataListMap size is less than expected.
        if (cloudBlobMetadataListMap.size() < blobIdList.size()) {
            Set<BlobId> missingBlobs = blobIdList.stream().filter(blobId -> !cloudBlobMetadataListMap.containsKey(blobId)).collect(Collectors.toSet());
            throw new StoreException("Some of the keys were missing in the cloud metadata store: " + missingBlobs, StoreErrorCodes.ID_Not_Found);
        }
        long currentTimeStamp = System.currentTimeMillis();
        // Validate cloud meta data, may throw StoreException with ID_Deleted, TTL_Expired and Authorization_Failure
        validateCloudMetadata(cloudBlobMetadataListMap, storeGetOptions, currentTimeStamp, ids);
        for (BlobId blobId : blobIdList) {
            CloudBlobMetadata blobMetadata = cloudBlobMetadataListMap.get(blobId.getID());
            // TODO: need to add ttlUpdated to CloudBlobMetadata so we can use it here
            // For now, set ttlUpdated = true for all permanent blobs, so the correct ttl
            // is applied by GetOperation.
            boolean ttlUpdated = blobMetadata.getExpirationTime() == Utils.Infinite_Time;
            boolean deleted = blobMetadata.getDeletionTime() != Utils.Infinite_Time;
            MessageInfo messageInfo = new MessageInfo(blobId, blobMetadata.getSize(), deleted, ttlUpdated, blobMetadata.isUndeleted(), blobMetadata.getExpirationTime(), null, (short) blobMetadata.getAccountId(), (short) blobMetadata.getContainerId(), getOperationTime(blobMetadata), blobMetadata.getLifeVersion());
            messageInfos.add(messageInfo);
            blobReadInfos.add(new CloudMessageReadSet.BlobReadInfo(blobMetadata, blobId));
        }
    } catch (CloudStorageException e) {
        if (e.getCause() instanceof StoreException) {
            throw (StoreException) e.getCause();
        } else {
            throw new StoreException(e, StoreErrorCodes.IOError);
        }
    }
    CloudMessageReadSet messageReadSet = new CloudMessageReadSet(blobReadInfos, this);
    return new StoreInfo(messageReadSet, messageInfos);
}

Example 62 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class CloudBlobStore method findEntriesSince.

@Override
public FindInfo findEntriesSince(FindToken token, long maxTotalSizeOfEntries, String hostname, String remoteReplicaPath) throws StoreException {
    try {
        FindResult findResult = requestAgent.doWithRetries(() -> cloudDestination.findEntriesSince(partitionId.toPathString(), token, maxTotalSizeOfEntries), "FindEntriesSince", partitionId.toPathString());
        if (findResult.getMetadataList().isEmpty()) {
            return new FindInfo(Collections.emptyList(), findResult.getUpdatedFindToken());
        }
        List<MessageInfo> messageEntries = new ArrayList<>();
        for (CloudBlobMetadata metadata : findResult.getMetadataList()) {
            messageEntries.add(getMessageInfoFromMetadata(metadata));
        }
        return new FindInfo(messageEntries, findResult.getUpdatedFindToken());
    } catch (CloudStorageException | IOException ex) {
        throw new StoreException(ex, StoreErrorCodes.IOError);
    }
}

Example 63 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method getMissingStoreKeys.

/**
 * Gets the missing store keys by comparing the messages from the remote node
 * @param replicaMetadataResponseInfo The response that contains the messages from the remote node
 * @param remoteNode The remote node from which replication needs to happen
 * @param remoteReplicaInfo The remote replica that contains information about the remote replica id
 * @return List of store keys that are missing from the local store
 * @throws StoreException
 */
private Set<StoreKey> getMissingStoreKeys(ReplicaMetadataResponseInfo replicaMetadataResponseInfo, DataNodeId remoteNode, RemoteReplicaInfo remoteReplicaInfo) throws StoreException {
    long startTime = SystemTime.getInstance().milliseconds();
    List<MessageInfo> messageInfoList = replicaMetadataResponseInfo.getMessageInfoList();
    List<StoreKey> storeKeysToCheck = new ArrayList<StoreKey>(messageInfoList.size());
    for (MessageInfo messageInfo : messageInfoList) {
        storeKeysToCheck.add(messageInfo.getStoreKey());
        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key from remote: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
    }
    Set<StoreKey> missingStoreKeys = remoteReplicaInfo.getLocalStore().findMissingKeys(storeKeysToCheck);
    for (StoreKey storeKey : missingStoreKeys) {
        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key missing id: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), storeKey);
    }
    replicationMetrics.updateCheckMissingKeysTime(SystemTime.getInstance().milliseconds() - startTime, replicatingFromRemoteColo, datacenterName);
    return missingStoreKeys;
}

Example 64 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method writeMessagesToLocalStoreAndAdvanceTokens.

/**
 * Writes the messages (if any) to the local stores from the remote stores for the missing keys, and advances tokens.
 * @param exchangeMetadataResponseList The list of metadata response from the remote node
 * @param getResponse The {@link GetResponse} that contains the missing messages. This may be null if there are no
 *                    missing messages to write as per the exchange metadata response. In that case this method will
 *                    simply advance the tokens for every store.
 * @param replicasToReplicatePerNode The list of remote replicas for the remote node
 * @param remoteNode The remote node from which replication needs to happen
 */
private void writeMessagesToLocalStoreAndAdvanceTokens(List<ExchangeMetadataResponse> exchangeMetadataResponseList, GetResponse getResponse, List<RemoteReplicaInfo> replicasToReplicatePerNode, DataNodeId remoteNode) throws IOException {
    int partitionResponseInfoIndex = 0;
    long totalBytesFixed = 0;
    long totalBlobsFixed = 0;
    long startTime = SystemTime.getInstance().milliseconds();
    for (int i = 0; i < exchangeMetadataResponseList.size(); i++) {
        ExchangeMetadataResponse exchangeMetadataResponse = exchangeMetadataResponseList.get(i);
        RemoteReplicaInfo remoteReplicaInfo = replicasToReplicatePerNode.get(i);
        if (exchangeMetadataResponse.serverErrorCode == ServerErrorCode.No_Error) {
            if (exchangeMetadataResponse.missingStoreKeys.size() > 0) {
                PartitionResponseInfo partitionResponseInfo = getResponse.getPartitionResponseInfoList().get(partitionResponseInfoIndex);
                responseHandler.onEvent(remoteReplicaInfo.getReplicaId(), partitionResponseInfo.getErrorCode());
                partitionResponseInfoIndex++;
                if (partitionResponseInfo.getPartition().compareTo(remoteReplicaInfo.getReplicaId().getPartitionId()) != 0) {
                    throw new IllegalStateException("The partition id from partitionResponseInfo " + partitionResponseInfo.getPartition() + " and from remoteReplicaInfo " + remoteReplicaInfo.getReplicaId().getPartitionId() + " are not the same");
                }
                if (partitionResponseInfo.getErrorCode() == ServerErrorCode.No_Error) {
                    try {
                        List<MessageInfo> messageInfoList = partitionResponseInfo.getMessageInfoList();
                        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Messages to fix: {} " + "Partition: {} Local mount path: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.missingStoreKeys, remoteReplicaInfo.getReplicaId().getPartitionId(), remoteReplicaInfo.getLocalReplicaId().getMountPath());
                        MessageFormatWriteSet writeset = null;
                        if (validateMessageStream) {
                            MessageSievingInputStream validMessageDetectionInputStream = new MessageSievingInputStream(getResponse.getInputStream(), messageInfoList, storeKeyFactory, metricRegistry);
                            if (validMessageDetectionInputStream.hasInvalidMessages()) {
                                replicationMetrics.incrementInvalidMessageError(partitionResponseInfo.getPartition());
                                logger.error("Out of " + (messageInfoList.size()) + " messages, " + (messageInfoList.size() - validMessageDetectionInputStream.getValidMessageInfoList().size()) + " invalid messages were found in message stream from " + remoteReplicaInfo.getReplicaId());
                            }
                            messageInfoList = validMessageDetectionInputStream.getValidMessageInfoList();
                            if (messageInfoList.size() == 0) {
                                logger.error("MessageInfoList is of size 0 as all messages are invalidated ");
                            } else {
                                writeset = new MessageFormatWriteSet(validMessageDetectionInputStream, messageInfoList, false);
                                remoteReplicaInfo.getLocalStore().put(writeset);
                            }
                        } else {
                            writeset = new MessageFormatWriteSet(getResponse.getInputStream(), messageInfoList, true);
                            remoteReplicaInfo.getLocalStore().put(writeset);
                        }
                        for (MessageInfo messageInfo : messageInfoList) {
                            totalBytesFixed += messageInfo.getSize();
                            logger.trace("Remote node: {} Thread name: {} Remote replica: {} Message replicated: {} Partition: {} " + "Local mount path: {} Message size: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey(), remoteReplicaInfo.getReplicaId().getPartitionId(), remoteReplicaInfo.getLocalReplicaId().getMountPath(), messageInfo.getSize());
                            if (notification != null) {
                                notification.onBlobReplicaCreated(dataNodeId.getHostname(), dataNodeId.getPort(), messageInfo.getStoreKey().getID(), BlobReplicaSourceType.REPAIRED);
                            }
                        }
                        totalBlobsFixed += messageInfoList.size();
                        remoteReplicaInfo.setToken(exchangeMetadataResponse.remoteToken);
                        remoteReplicaInfo.setLocalLagFromRemoteInBytes(exchangeMetadataResponse.localLagFromRemoteInBytes);
                        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token after speaking to remote node: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.remoteToken);
                    } catch (StoreException e) {
                        if (e.getErrorCode() != StoreErrorCodes.Already_Exist) {
                            replicationMetrics.updateLocalStoreError(remoteReplicaInfo.getReplicaId());
                            logger.error("Remote node: " + remoteNode + " Thread name: " + threadName + " Remote replica: " + remoteReplicaInfo.getReplicaId(), e);
                        }
                    }
                } else if (partitionResponseInfo.getErrorCode() == ServerErrorCode.Blob_Deleted) {
                    replicationMetrics.blobDeletedOnGetCount.inc();
                    logger.trace("One of the blobs to GET is deleted: Remote node: {} Thread name: {} Remote replica: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId());
                } else {
                    replicationMetrics.updateGetRequestError(remoteReplicaInfo.getReplicaId());
                    logger.error("Remote node: {} Thread name: {} Remote replica: {} Server error: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), partitionResponseInfo.getErrorCode());
                }
            } else {
                // There are no missing keys. We just advance the token
                remoteReplicaInfo.setToken(exchangeMetadataResponse.remoteToken);
                remoteReplicaInfo.setLocalLagFromRemoteInBytes(exchangeMetadataResponse.localLagFromRemoteInBytes);
                logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token after speaking to remote node: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), exchangeMetadataResponse.remoteToken);
            }
        }
    }
    long batchStoreWriteTime = SystemTime.getInstance().milliseconds() - startTime;
    replicationMetrics.updateBatchStoreWriteTime(batchStoreWriteTime, totalBytesFixed, totalBlobsFixed, replicatingFromRemoteColo, replicatingOverSsl, datacenterName);
}

Example 65 with MessageInfo

use of com.github.ambry.store.MessageInfo in project ambry by linkedin.

the class ReplicaThread method processReplicaMetadataResponse.

/**
 * Takes the missing keys and the message list from the remote store and identifies messages that are deleted
 * on the remote store and updates them locally. Also, if the message that is missing is deleted in the remote
 * store, we remove the message from the list of missing keys
 * @param missingStoreKeys The list of keys missing from the local store
 * @param replicaMetadataResponseInfo The replica metadata response from the remote store
 * @param remoteReplicaInfo The remote replica that is being replicated from
 * @param remoteNode The remote node from which replication needs to happen
 * @throws IOException
 * @throws StoreException
 * @throws MessageFormatException
 */
private void processReplicaMetadataResponse(Set<StoreKey> missingStoreKeys, ReplicaMetadataResponseInfo replicaMetadataResponseInfo, RemoteReplicaInfo remoteReplicaInfo, DataNodeId remoteNode) throws IOException, StoreException, MessageFormatException {
    long startTime = SystemTime.getInstance().milliseconds();
    List<MessageInfo> messageInfoList = replicaMetadataResponseInfo.getMessageInfoList();
    for (MessageInfo messageInfo : messageInfoList) {
        BlobId blobId = (BlobId) messageInfo.getStoreKey();
        if (remoteReplicaInfo.getLocalReplicaId().getPartitionId().compareTo(blobId.getPartition()) != 0) {
            throw new IllegalStateException("Blob id is not in the expected partition Actual partition " + blobId.getPartition() + " Expected partition " + remoteReplicaInfo.getLocalReplicaId().getPartitionId());
        }
        if (!missingStoreKeys.contains(messageInfo.getStoreKey())) {
            // deleted yet locally
            if (messageInfo.isDeleted() && !remoteReplicaInfo.getLocalStore().isKeyDeleted(messageInfo.getStoreKey())) {
                MessageFormatInputStream deleteStream = new DeleteMessageFormatInputStream(messageInfo.getStoreKey(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs());
                MessageInfo info = new MessageInfo(messageInfo.getStoreKey(), deleteStream.getSize(), true, messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs());
                ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
                infoList.add(info);
                MessageFormatWriteSet writeset = new MessageFormatWriteSet(deleteStream, infoList, false);
                try {
                    remoteReplicaInfo.getLocalStore().delete(writeset);
                    logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key deleted. mark for deletion id: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
                } catch (StoreException e) {
                    // messages are received from different replicas around the same time.
                    if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
                        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key already deleted: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
                    } else {
                        throw e;
                    }
                }
                // as long as the Delete is guaranteed to have taken effect locally.
                if (notification != null) {
                    notification.onBlobReplicaDeleted(dataNodeId.getHostname(), dataNodeId.getPort(), messageInfo.getStoreKey().getID(), BlobReplicaSourceType.REPAIRED);
                }
            }
        } else {
            if (messageInfo.isDeleted()) {
                // if the key is not present locally and if the remote replica has the message in deleted state,
                // it is not considered missing locally.
                missingStoreKeys.remove(messageInfo.getStoreKey());
                logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key in deleted state remotely: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
                // as long as the Delete is guaranteed to have taken effect locally.
                if (notification != null) {
                    notification.onBlobReplicaDeleted(dataNodeId.getHostname(), dataNodeId.getPort(), messageInfo.getStoreKey().getID(), BlobReplicaSourceType.REPAIRED);
                }
            } else if (messageInfo.isExpired()) {
                // if the key is not present locally and if the remote replica has the key as expired,
                // it is not considered missing locally.
                missingStoreKeys.remove(messageInfo.getStoreKey());
                logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key in expired state remotely {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
            }
        }
    }
    if (replicatingFromRemoteColo) {
        replicationMetrics.interColoProcessMetadataResponseTime.get(datacenterName).update(SystemTime.getInstance().milliseconds() - startTime);
    } else {
        replicationMetrics.intraColoProcessMetadataResponseTime.update(SystemTime.getInstance().milliseconds() - startTime);
    }
}