Examples with StoreException com.github.ambry.store.StoreException used on opensource projects

Example 36 with StoreException

use of com.github.ambry.store.StoreException in project ambry by linkedin.

the class ReplicaThread method processReplicaMetadataResponse.

/**
 * Takes the missing keys and the message list from the remote store and identifies messages that are deleted
 * on the remote store and updates them locally. Also, if the message that is missing is deleted in the remote
 * store, we remove the message from the list of missing keys
 * @param missingStoreKeys The list of keys missing from the local store
 * @param replicaMetadataResponseInfo The replica metadata response from the remote store
 * @param remoteReplicaInfo The remote replica that is being replicated from
 * @param remoteNode The remote node from which replication needs to happen
 * @throws IOException
 * @throws StoreException
 * @throws MessageFormatException
 */
private void processReplicaMetadataResponse(Set<StoreKey> missingStoreKeys, ReplicaMetadataResponseInfo replicaMetadataResponseInfo, RemoteReplicaInfo remoteReplicaInfo, DataNodeId remoteNode) throws IOException, StoreException, MessageFormatException {
    long startTime = SystemTime.getInstance().milliseconds();
    List<MessageInfo> messageInfoList = replicaMetadataResponseInfo.getMessageInfoList();
    for (MessageInfo messageInfo : messageInfoList) {
        BlobId blobId = (BlobId) messageInfo.getStoreKey();
        if (remoteReplicaInfo.getLocalReplicaId().getPartitionId().compareTo(blobId.getPartition()) != 0) {
            throw new IllegalStateException("Blob id is not in the expected partition Actual partition " + blobId.getPartition() + " Expected partition " + remoteReplicaInfo.getLocalReplicaId().getPartitionId());
        }
        if (!missingStoreKeys.contains(messageInfo.getStoreKey())) {
            // deleted yet locally
            if (messageInfo.isDeleted() && !remoteReplicaInfo.getLocalStore().isKeyDeleted(messageInfo.getStoreKey())) {
                MessageFormatInputStream deleteStream = new DeleteMessageFormatInputStream(messageInfo.getStoreKey(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs());
                MessageInfo info = new MessageInfo(messageInfo.getStoreKey(), deleteStream.getSize(), true, messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs());
                ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
                infoList.add(info);
                MessageFormatWriteSet writeset = new MessageFormatWriteSet(deleteStream, infoList, false);
                try {
                    remoteReplicaInfo.getLocalStore().delete(writeset);
                    logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key deleted. mark for deletion id: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
                } catch (StoreException e) {
                    // messages are received from different replicas around the same time.
                    if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
                        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key already deleted: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
                    } else {
                        throw e;
                    }
                }
                // as long as the Delete is guaranteed to have taken effect locally.
                if (notification != null) {
                    notification.onBlobReplicaDeleted(dataNodeId.getHostname(), dataNodeId.getPort(), messageInfo.getStoreKey().getID(), BlobReplicaSourceType.REPAIRED);
                }
            }
        } else {
            if (messageInfo.isDeleted()) {
                // if the key is not present locally and if the remote replica has the message in deleted state,
                // it is not considered missing locally.
                missingStoreKeys.remove(messageInfo.getStoreKey());
                logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key in deleted state remotely: {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
                // as long as the Delete is guaranteed to have taken effect locally.
                if (notification != null) {
                    notification.onBlobReplicaDeleted(dataNodeId.getHostname(), dataNodeId.getPort(), messageInfo.getStoreKey().getID(), BlobReplicaSourceType.REPAIRED);
                }
            } else if (messageInfo.isExpired()) {
                // if the key is not present locally and if the remote replica has the key as expired,
                // it is not considered missing locally.
                missingStoreKeys.remove(messageInfo.getStoreKey());
                logger.trace("Remote node: {} Thread name: {} Remote replica: {} Key in expired state remotely {}", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), messageInfo.getStoreKey());
            }
        }
    }
    if (replicatingFromRemoteColo) {
        replicationMetrics.interColoProcessMetadataResponseTime.get(datacenterName).update(SystemTime.getInstance().milliseconds() - startTime);
    } else {
        replicationMetrics.intraColoProcessMetadataResponseTime.update(SystemTime.getInstance().milliseconds() - startTime);
    }
}

Example 37 with StoreException

use of com.github.ambry.store.StoreException in project ambry by linkedin.

the class AmbryRequests method handleDeleteRequest.

public void handleDeleteRequest(Request request) throws IOException, InterruptedException {
    DeleteRequest deleteRequest = DeleteRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.deleteBlobRequestQueueTimeInMs.update(requestQueueTime);
    metrics.deleteBlobRequestRate.mark();
    long startTime = SystemTime.getInstance().milliseconds();
    DeleteResponse response = null;
    try {
        ServerErrorCode error = validateRequest(deleteRequest.getBlobId().getPartition(), RequestOrResponseType.DeleteRequest);
        if (error != ServerErrorCode.No_Error) {
            logger.error("Validating delete request failed with error {} for request {}", error, deleteRequest);
            response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), error);
        } else {
            MessageFormatInputStream stream = new DeleteMessageFormatInputStream(deleteRequest.getBlobId(), deleteRequest.getAccountId(), deleteRequest.getContainerId(), deleteRequest.getDeletionTimeInMs());
            MessageInfo info = new MessageInfo(deleteRequest.getBlobId(), stream.getSize(), deleteRequest.getAccountId(), deleteRequest.getContainerId(), deleteRequest.getDeletionTimeInMs());
            ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
            infoList.add(info);
            MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
            Store storeToDelete = storageManager.getStore(deleteRequest.getBlobId().getPartition());
            storeToDelete.delete(writeset);
            response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), ServerErrorCode.No_Error);
            if (notification != null) {
                notification.onBlobReplicaDeleted(currentNode.getHostname(), currentNode.getPort(), deleteRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
            }
        }
    } catch (StoreException e) {
        boolean logInErrorLevel = false;
        if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
            metrics.idNotFoundError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
            metrics.ttlExpiredError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
            metrics.idDeletedError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
            metrics.deleteAuthorizationFailure.inc();
        } else {
            logInErrorLevel = true;
            metrics.unExpectedStoreDeleteError.inc();
        }
        if (logInErrorLevel) {
            logger.error("Store exception on a delete with error code {} for request {}", e.getErrorCode(), deleteRequest, e);
        } else {
            logger.trace("Store exception on a delete with error code {} for request {}", e.getErrorCode(), deleteRequest, e);
        }
        response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
    } catch (Exception e) {
        logger.error("Unknown exception for delete request " + deleteRequest, e);
        response = new DeleteResponse(deleteRequest.getCorrelationId(), deleteRequest.getClientId(), ServerErrorCode.Unknown_Error);
        metrics.unExpectedStoreDeleteError.inc();
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", deleteRequest, response, processingTime);
        metrics.deleteBlobProcessingTimeInMs.update(processingTime);
    }
    requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.deleteBlobResponseQueueTimeInMs, metrics.deleteBlobSendTimeInMs, metrics.deleteBlobTotalTimeInMs, null, null, totalTimeSpent));
}

Example 38 with StoreException

use of com.github.ambry.store.StoreException in project ambry by linkedin.

the class AmbryRequests method handlePutRequest.

public void handlePutRequest(Request request) throws IOException, InterruptedException {
    PutRequest.ReceivedPutRequest receivedRequest = PutRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.putBlobRequestQueueTimeInMs.update(requestQueueTime);
    metrics.putBlobRequestRate.mark();
    long startTime = SystemTime.getInstance().milliseconds();
    PutResponse response = null;
    try {
        ServerErrorCode error = validateRequest(receivedRequest.getBlobId().getPartition(), RequestOrResponseType.PutRequest);
        if (error != ServerErrorCode.No_Error) {
            logger.error("Validating put request failed with error {} for request {}", error, receivedRequest);
            response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), error);
        } else {
            MessageFormatInputStream stream = new PutMessageFormatInputStream(receivedRequest.getBlobId(), receivedRequest.getBlobEncryptionKey(), receivedRequest.getBlobProperties(), receivedRequest.getUsermetadata(), receivedRequest.getBlobStream(), receivedRequest.getBlobSize(), receivedRequest.getBlobType());
            MessageInfo info = new MessageInfo(receivedRequest.getBlobId(), stream.getSize(), false, Utils.addSecondsToEpochTime(receivedRequest.getBlobProperties().getCreationTimeInMs(), receivedRequest.getBlobProperties().getTimeToLiveInSeconds()), receivedRequest.getCrc(), receivedRequest.getBlobProperties().getAccountId(), receivedRequest.getBlobProperties().getContainerId(), receivedRequest.getBlobProperties().getCreationTimeInMs());
            ArrayList<MessageInfo> infoList = new ArrayList<MessageInfo>();
            infoList.add(info);
            MessageFormatWriteSet writeset = new MessageFormatWriteSet(stream, infoList, false);
            Store storeToPut = storageManager.getStore(receivedRequest.getBlobId().getPartition());
            storeToPut.put(writeset);
            response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.No_Error);
            metrics.blobSizeInBytes.update(receivedRequest.getBlobSize());
            metrics.blobUserMetadataSizeInBytes.update(receivedRequest.getUsermetadata().limit());
            if (notification != null) {
                notification.onBlobReplicaCreated(currentNode.getHostname(), currentNode.getPort(), receivedRequest.getBlobId().getID(), BlobReplicaSourceType.PRIMARY);
            }
        }
    } catch (StoreException e) {
        logger.error("Store exception on a put with error code " + e.getErrorCode() + " for request " + receivedRequest, e);
        if (e.getErrorCode() == StoreErrorCodes.Already_Exist) {
            metrics.idAlreadyExistError.inc();
        } else if (e.getErrorCode() == StoreErrorCodes.IOError) {
            metrics.storeIOError.inc();
        } else {
            metrics.unExpectedStorePutError.inc();
        }
        response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
    } catch (Exception e) {
        logger.error("Unknown exception on a put for request " + receivedRequest, e);
        response = new PutResponse(receivedRequest.getCorrelationId(), receivedRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", receivedRequest, response, processingTime);
        metrics.putBlobProcessingTimeInMs.update(processingTime);
        metrics.updatePutBlobProcessingTimeBySize(receivedRequest.getBlobSize(), processingTime);
    }
    sendPutResponse(requestResponseChannel, response, request, metrics.putBlobResponseQueueTimeInMs, metrics.putBlobSendTimeInMs, metrics.putBlobTotalTimeInMs, totalTimeSpent, receivedRequest.getBlobSize(), metrics);
}

Example 39 with StoreException

use of com.github.ambry.store.StoreException in project ambry by linkedin.

the class StatsManager method fetchSnapshot.

/**
 * Fetch the {@link StatsSnapshot} for the given {@link PartitionId}.
 * @param partitionId the {@link PartitionId} to try to fetch the {@link StatsSnapshot} from
 * @param unreachableStores a list of partitionIds to keep track of the unreachable stores (partitions)
 * @return
 */
StatsSnapshot fetchSnapshot(PartitionId partitionId, List<String> unreachableStores) {
    StatsSnapshot statsSnapshot = null;
    Store store = storageManager.getStore(partitionId);
    if (store == null) {
        unreachableStores.add(partitionId.toString());
    } else {
        try {
            statsSnapshot = store.getStoreStats().getStatsSnapshot(time.milliseconds());
        } catch (StoreException e) {
            logger.error("StoreException on fetching stats snapshot for store {}", store, e);
            unreachableStores.add(partitionId.toString());
        }
    }
    return statsSnapshot;
}

Example 40 with StoreException

use of com.github.ambry.store.StoreException in project ambry by linkedin.

the class CloudBlobStore method findMissingKeys.

@Override
public Set<StoreKey> findMissingKeys(List<StoreKey> keys) throws StoreException {
    checkStarted();
    // Check existence of keys in cloud metadata
    // Note that it is ok to refer cache here, because all we are doing is eliminating blobs that were seen before and
    // we don't care about the state of the blob.
    // TODO Fix corner case where a blob is deleted in cache, and has been compacted. Ideally it should show as missing.
    List<BlobId> blobIdQueryList = keys.stream().filter(key -> !checkCacheState(key.getID())).map(key -> (BlobId) key).collect(Collectors.toList());
    if (blobIdQueryList.isEmpty()) {
        // Cool, the cache did its job and eliminated a possibly expensive query to cloud!
        return Collections.emptySet();
    }
    try {
        Set<String> foundSet = requestAgent.doWithRetries(() -> cloudDestination.getBlobMetadata(blobIdQueryList), "FindMissingKeys", partitionId.toPathString()).keySet();
        // return input keys - cached keys - keys returned by query
        return keys.stream().filter(key -> !foundSet.contains(key.getID())).filter(key -> !recentBlobCache.containsKey(key.getID())).collect(Collectors.toSet());
    } catch (CloudStorageException ex) {
        throw new StoreException(ex, StoreErrorCodes.IOError);
    }
}