Examples with ReplicaMetadataRequest com.github.ambry.protocol.ReplicaMetadataRequest used on opensource projects

Example 1 with ReplicaMetadataRequest

use of com.github.ambry.protocol.ReplicaMetadataRequest in project ambry by linkedin.

the class AmbryRequests method handleReplicaMetadataRequest.

public void handleReplicaMetadataRequest(Request request) throws IOException, InterruptedException {
    ReplicaMetadataRequest replicaMetadataRequest = ReplicaMetadataRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap, findTokenFactory);
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.replicaMetadataRequestQueueTimeInMs.update(requestQueueTime);
    metrics.replicaMetadataRequestRate.mark();
    List<ReplicaMetadataRequestInfo> replicaMetadataRequestInfoList = replicaMetadataRequest.getReplicaMetadataRequestInfoList();
    int partitionCnt = replicaMetadataRequestInfoList.size();
    long startTimeInMs = SystemTime.getInstance().milliseconds();
    ReplicaMetadataResponse response = null;
    try {
        List<ReplicaMetadataResponseInfo> replicaMetadataResponseList = new ArrayList<ReplicaMetadataResponseInfo>(partitionCnt);
        for (ReplicaMetadataRequestInfo replicaMetadataRequestInfo : replicaMetadataRequestInfoList) {
            long partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
            PartitionId partitionId = replicaMetadataRequestInfo.getPartitionId();
            ServerErrorCode error = validateRequest(partitionId, RequestOrResponseType.ReplicaMetadataRequest);
            logger.trace("{} Time used to validate metadata request: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
            if (error != ServerErrorCode.No_Error) {
                logger.error("Validating replica metadata request failed with error {} for partition {}", error, partitionId);
                ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, error);
                replicaMetadataResponseList.add(replicaMetadataResponseInfo);
            } else {
                try {
                    FindToken findToken = replicaMetadataRequestInfo.getToken();
                    String hostName = replicaMetadataRequestInfo.getHostName();
                    String replicaPath = replicaMetadataRequestInfo.getReplicaPath();
                    Store store = storageManager.getStore(partitionId);
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    FindInfo findInfo = store.findEntriesSince(findToken, replicaMetadataRequest.getMaxTotalSizeOfEntriesInBytes());
                    logger.trace("{} Time used to find entry since: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    replicationManager.updateTotalBytesReadByRemoteReplica(partitionId, hostName, replicaPath, findInfo.getFindToken().getBytesRead());
                    logger.trace("{} Time used to update total bytes read: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    long remoteReplicaLagInBytes = replicationManager.getRemoteReplicaLagFromLocalInBytes(partitionId, hostName, replicaPath);
                    logger.trace("{} Time used to get remote replica lag in bytes: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, findInfo.getFindToken(), findInfo.getMessageEntries(), remoteReplicaLagInBytes);
                    replicaMetadataResponseList.add(replicaMetadataResponseInfo);
                } catch (StoreException e) {
                    logger.error("Store exception on a replica metadata request with error code " + e.getErrorCode() + " for partition " + partitionId, e);
                    if (e.getErrorCode() == StoreErrorCodes.IOError) {
                        metrics.storeIOError.inc();
                    } else {
                        metrics.unExpectedStoreFindEntriesError.inc();
                    }
                    ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
                    replicaMetadataResponseList.add(replicaMetadataResponseInfo);
                }
            }
        }
        response = new ReplicaMetadataResponse(replicaMetadataRequest.getCorrelationId(), replicaMetadataRequest.getClientId(), ServerErrorCode.No_Error, replicaMetadataResponseList);
    } catch (Exception e) {
        logger.error("Unknown exception for request " + replicaMetadataRequest, e);
        response = new ReplicaMetadataResponse(replicaMetadataRequest.getCorrelationId(), replicaMetadataRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTimeInMs;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", replicaMetadataRequest, response, processingTime);
        logger.trace("{} {} processingTime {}", replicaMetadataRequest, response, processingTime);
        metrics.replicaMetadataRequestProcessingTimeInMs.update(processingTime);
    }
    requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.replicaMetadataResponseQueueTimeInMs, metrics.replicaMetadataSendTimeInMs, metrics.replicaMetadataTotalTimeInMs, null, null, totalTimeSpent));
}

Example 2 with ReplicaMetadataRequest

use of com.github.ambry.protocol.ReplicaMetadataRequest in project ambry by linkedin.

the class AmbryRequestsTest method sendAndVerifyOperationRequest.

/**
 * Sends and verifies that an operation specific request works correctly.
 * @param requestType the type of the request to send.
 * @param ids the partitionIds to send requests for.
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
 *                          response in the constituents rather than the actual response ({@link GetResponse} and
 *                          {@link ReplicaMetadataResponse}).
 * @throws InterruptedException
 * @throws IOException
 */
private void sendAndVerifyOperationRequest(RequestOrResponseType requestType, List<? extends PartitionId> ids, ServerErrorCode expectedErrorCode) throws InterruptedException, IOException {
    for (PartitionId id : ids) {
        int correlationId = TestUtils.RANDOM.nextInt();
        String clientId = UtilsTest.getRandomString(10);
        BlobId blobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMapUtils.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), id, false);
        RequestOrResponse request;
        switch(requestType) {
            case PutRequest:
                BlobProperties properties = new BlobProperties(0, "serviceId", blobId.getAccountId(), blobId.getAccountId(), false);
                request = new PutRequest(correlationId, clientId, blobId, properties, ByteBuffer.allocate(0), ByteBuffer.allocate(0), 0, BlobType.DataBlob, null);
                break;
            case DeleteRequest:
                request = new DeleteRequest(correlationId, clientId, blobId, SystemTime.getInstance().milliseconds());
                break;
            case GetRequest:
                PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(id, Collections.singletonList(blobId));
                request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
                break;
            case ReplicaMetadataRequest:
                ReplicaMetadataRequestInfo rRequestInfo = new ReplicaMetadataRequestInfo(id, FIND_TOKEN_FACTORY.getNewFindToken(), "localhost", "/tmp");
                request = new ReplicaMetadataRequest(correlationId, clientId, Collections.singletonList(rRequestInfo), Long.MAX_VALUE);
                break;
            default:
                throw new IllegalArgumentException(requestType + " not supported by this function");
        }
        storageManager.resetStore();
        Response response = sendRequestGetResponse(request, requestType == RequestOrResponseType.GetRequest || requestType == RequestOrResponseType.ReplicaMetadataRequest ? ServerErrorCode.No_Error : expectedErrorCode);
        if (expectedErrorCode.equals(ServerErrorCode.No_Error)) {
            assertEquals("Operation received at the store not as expected", requestType, MockStorageManager.operationReceived);
        }
        if (requestType == RequestOrResponseType.GetRequest) {
            GetResponse getResponse = (GetResponse) response;
            for (PartitionResponseInfo info : getResponse.getPartitionResponseInfoList()) {
                assertEquals("Error code does not match expected", expectedErrorCode, info.getErrorCode());
            }
        } else if (requestType == RequestOrResponseType.ReplicaMetadataRequest) {
            ReplicaMetadataResponse replicaMetadataResponse = (ReplicaMetadataResponse) response;
            for (ReplicaMetadataResponseInfo info : replicaMetadataResponse.getReplicaMetadataResponseInfoList()) {
                assertEquals("Error code does not match expected", expectedErrorCode, info.getError());
            }
        }
    }
}

Example 3 with ReplicaMetadataRequest

use of com.github.ambry.protocol.ReplicaMetadataRequest in project ambry by linkedin.

the class AmbryServerRequestsTest method sendAndVerifyOperationRequest.

/**
 * Sends and verifies that an operation specific request works correctly.
 * @param requestType the type of the request to send.
 * @param ids the partitionIds to send requests for.
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
 *                          response in the constituents rather than the actual response ({@link GetResponse} and
 *                          {@link ReplicaMetadataResponse}).
 * @param forceCheckOpReceived if {@code true}, checks the operation received at the {@link Store} even if
 *                             there is an error expected. Always checks op received if {@code expectedErrorCode} is
 *                             {@link ServerErrorCode#No_Error}. Skips the check otherwise.
 * @param clientIdStr the clientId string to construct request. if null, generate a random string as clientId.
 * @throws InterruptedException
 * @throws IOException
 * @return a list of {@link Response}(s) associated with given partition ids.
 */
private List<Response> sendAndVerifyOperationRequest(RequestOrResponseType requestType, List<? extends PartitionId> ids, ServerErrorCode expectedErrorCode, Boolean forceCheckOpReceived, String clientIdStr) throws InterruptedException, IOException {
    List<Response> responses = new ArrayList<>();
    for (PartitionId id : ids) {
        int correlationId = TestUtils.RANDOM.nextInt();
        String clientId = clientIdStr == null ? TestUtils.getRandomString(10) : clientIdStr;
        BlobId originalBlobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMap.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), id, false, BlobId.BlobDataType.DATACHUNK);
        BlobId convertedBlobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.CRAFTED, ClusterMap.UNKNOWN_DATACENTER_ID, originalBlobId.getAccountId(), originalBlobId.getContainerId(), id, false, BlobId.BlobDataType.DATACHUNK);
        conversionMap.put(originalBlobId, convertedBlobId);
        validKeysInStore.add(convertedBlobId);
        RequestOrResponse request;
        switch(requestType) {
            case PutRequest:
                BlobProperties properties = new BlobProperties(0, "serviceId", originalBlobId.getAccountId(), originalBlobId.getAccountId(), false);
                request = new PutRequest(correlationId, clientId, originalBlobId, properties, ByteBuffer.allocate(0), Unpooled.wrappedBuffer(ByteBuffer.allocate(0)), 0, BlobType.DataBlob, null);
                break;
            case DeleteRequest:
                request = new DeleteRequest(correlationId, clientId, originalBlobId, SystemTime.getInstance().milliseconds());
                break;
            case UndeleteRequest:
                request = new UndeleteRequest(correlationId, clientId, originalBlobId, SystemTime.getInstance().milliseconds());
                break;
            case GetRequest:
                PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(id, Collections.singletonList(originalBlobId));
                request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
                break;
            case ReplicaMetadataRequest:
                ReplicaMetadataRequestInfo rRequestInfo = new ReplicaMetadataRequestInfo(id, findTokenHelper.getFindTokenFactoryFromReplicaType(ReplicaType.DISK_BACKED).getNewFindToken(), "localhost", "/tmp", ReplicaType.DISK_BACKED, replicationConfig.replicaMetadataRequestVersion);
                request = new ReplicaMetadataRequest(correlationId, clientId, Collections.singletonList(rRequestInfo), Long.MAX_VALUE, replicationConfig.replicaMetadataRequestVersion);
                break;
            case TtlUpdateRequest:
                request = new TtlUpdateRequest(correlationId, clientId, originalBlobId, Utils.Infinite_Time, SystemTime.getInstance().milliseconds());
                break;
            default:
                throw new IllegalArgumentException(requestType + " not supported by this function");
        }
        responses.add(sendAndVerifyOperationRequest(request, expectedErrorCode, forceCheckOpReceived));
    }
    return responses;
}

Example 4 with ReplicaMetadataRequest

use of com.github.ambry.protocol.ReplicaMetadataRequest in project ambry by linkedin.

the class ReplicaThread method getReplicaMetadataResponse.

/**
 * Gets the replica metadata response for a list of remote replicas on a given remote data node
 * @param replicasToReplicatePerNode The list of remote replicas for a node
 * @param connectedChannel The connection channel to the node
 * @param remoteNode The remote node from which replication needs to happen
 * @return ReplicaMetadataResponse, the response from replica metadata request to remote node
 * @throws ReplicationException
 * @throws IOException
 */
ReplicaMetadataResponse getReplicaMetadataResponse(List<RemoteReplicaInfo> replicasToReplicatePerNode, ConnectedChannel connectedChannel, DataNodeId remoteNode) throws ReplicationException, IOException {
    long replicaMetadataRequestStartTime = time.milliseconds();
    List<ReplicaMetadataRequestInfo> replicaMetadataRequestInfoList = new ArrayList<ReplicaMetadataRequestInfo>();
    for (RemoteReplicaInfo remoteReplicaInfo : replicasToReplicatePerNode) {
        ReplicaMetadataRequestInfo replicaMetadataRequestInfo = new ReplicaMetadataRequestInfo(remoteReplicaInfo.getReplicaId().getPartitionId(), remoteReplicaInfo.getToken(), dataNodeId.getHostname(), remoteReplicaInfo.getLocalReplicaId().getReplicaPath(), remoteReplicaInfo.getReplicaId().getReplicaType(), replicationConfig.replicaMetadataRequestVersion);
        replicaMetadataRequestInfoList.add(replicaMetadataRequestInfo);
        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token going to be sent to remote: {} ", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), remoteReplicaInfo.getToken());
    }
    ChannelOutput channelOutput = null;
    try {
        ReplicaMetadataRequest request = new ReplicaMetadataRequest(correlationIdGenerator.incrementAndGet(), "replication-metadata-" + dataNodeId.getHostname() + "[" + dataNodeId.getDatacenterName() + "]", replicaMetadataRequestInfoList, replicationConfig.replicationFetchSizeInBytes, replicationConfig.replicaMetadataRequestVersion);
        channelOutput = connectedChannel.sendAndReceive(request);
        logger.trace("Remote node: {} Thread name: {} Remote replicas: {} Stream size after deserialization: {} ", remoteNode, threadName, replicasToReplicatePerNode, channelOutput.getInputStream().available());
        ReplicaMetadataResponse response = ReplicaMetadataResponse.readFrom(channelOutput.getInputStream(), findTokenHelper, clusterMap);
        long metadataRequestTime = time.milliseconds() - replicaMetadataRequestStartTime;
        replicationMetrics.updateMetadataRequestTime(metadataRequestTime, replicatingFromRemoteColo, replicatingOverSsl, datacenterName);
        if (response.getError() != ServerErrorCode.No_Error || response.getReplicaMetadataResponseInfoList().size() != replicasToReplicatePerNode.size()) {
            int replicaMetadataResponseInfoListSize = response.getReplicaMetadataResponseInfoList() == null ? 0 : response.getReplicaMetadataResponseInfoList().size();
            logger.error("Remote node: {} Thread name: {} Remote replicas: {} Replica metadata response error: {} ReplicaMetadataResponseInfoListSize: {} ReplicasToReplicatePerNodeSize: {}", remoteNode, threadName, replicasToReplicatePerNode, response.getError(), replicaMetadataResponseInfoListSize, replicasToReplicatePerNode.size());
            throw new ReplicationException("Replica Metadata Response Error " + response.getError());
        }
        return response;
    } catch (Exception e) {
        responseHandler.onEvent(replicasToReplicatePerNode.get(0).getReplicaId(), e);
        throw e;
    } finally {
        if (channelOutput != null && (channelOutput.getInputStream() instanceof NettyByteBufDataInputStream)) {
            // Release buffer if and only if the inputStream is NettyByteBuf based.
            ((NettyByteBufDataInputStream) channelOutput.getInputStream()).getBuffer().release();
        }
    }
}