Examples with PartitionResponseInfo com.github.ambry.protocol.PartitionResponseInfo used on opensource projects

Example 1 with PartitionResponseInfo

use of com.github.ambry.protocol.PartitionResponseInfo in project ambry by linkedin.

the class AmbryRequests method handleGetRequest.

public void handleGetRequest(Request request) throws IOException, InterruptedException {
    GetRequest getRequest = GetRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
    Histogram responseQueueTime = null;
    Histogram responseSendTime = null;
    Histogram responseTotalTime = null;
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
        metrics.getBlobRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobRequestRate.mark();
        responseQueueTime = metrics.getBlobResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobSendTimeInMs;
        responseTotalTime = metrics.getBlobTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
        metrics.getBlobPropertiesRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobPropertiesRequestRate.mark();
        responseQueueTime = metrics.getBlobPropertiesResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobPropertiesSendTimeInMs;
        responseTotalTime = metrics.getBlobPropertiesTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
        metrics.getBlobUserMetadataRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobUserMetadataRequestRate.mark();
        responseQueueTime = metrics.getBlobUserMetadataResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobUserMetadataSendTimeInMs;
        responseTotalTime = metrics.getBlobUserMetadataTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
        metrics.getBlobInfoRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobInfoRequestRate.mark();
        responseQueueTime = metrics.getBlobInfoResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobInfoSendTimeInMs;
        responseTotalTime = metrics.getBlobInfoTotalTimeInMs;
    } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
        metrics.getBlobAllRequestQueueTimeInMs.update(requestQueueTime);
        metrics.getBlobAllRequestRate.mark();
        responseQueueTime = metrics.getBlobAllResponseQueueTimeInMs;
        responseSendTime = metrics.getBlobAllSendTimeInMs;
        responseTotalTime = metrics.getBlobAllTotalTimeInMs;
    }
    long startTime = SystemTime.getInstance().milliseconds();
    GetResponse response = null;
    try {
        List<Send> messagesToSendList = new ArrayList<Send>(getRequest.getPartitionInfoList().size());
        List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<PartitionResponseInfo>(getRequest.getPartitionInfoList().size());
        for (PartitionRequestInfo partitionRequestInfo : getRequest.getPartitionInfoList()) {
            ServerErrorCode error = validateRequest(partitionRequestInfo.getPartition(), RequestOrResponseType.GetRequest);
            if (error != ServerErrorCode.No_Error) {
                logger.error("Validating get request failed for partition {} with error {}", partitionRequestInfo.getPartition(), error);
                PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), error);
                partitionResponseInfoList.add(partitionResponseInfo);
            } else {
                try {
                    Store storeToGet = storageManager.getStore(partitionRequestInfo.getPartition());
                    EnumSet<StoreGetOptions> storeGetOptions = EnumSet.noneOf(StoreGetOptions.class);
                    // Currently only one option is supported.
                    if (getRequest.getGetOption() == GetOption.Include_Expired_Blobs) {
                        storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Expired);
                    }
                    if (getRequest.getGetOption() == GetOption.Include_Deleted_Blobs) {
                        storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted);
                    }
                    if (getRequest.getGetOption() == GetOption.Include_All) {
                        storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted, StoreGetOptions.Store_Include_Expired);
                    }
                    StoreInfo info = storeToGet.get(partitionRequestInfo.getBlobIds(), storeGetOptions);
                    MessageFormatSend blobsToSend = new MessageFormatSend(info.getMessageReadSet(), getRequest.getMessageFormatFlag(), messageFormatMetrics, storeKeyFactory);
                    PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), info.getMessageReadSetInfo(), blobsToSend.getMessageMetadataList());
                    messagesToSendList.add(blobsToSend);
                    partitionResponseInfoList.add(partitionResponseInfo);
                } catch (StoreException e) {
                    boolean logInErrorLevel = false;
                    if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
                        metrics.idNotFoundError.inc();
                    } else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
                        metrics.ttlExpiredError.inc();
                    } else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
                        metrics.idDeletedError.inc();
                    } else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
                        metrics.getAuthorizationFailure.inc();
                    } else {
                        metrics.unExpectedStoreGetError.inc();
                        logInErrorLevel = true;
                    }
                    if (logInErrorLevel) {
                        logger.error("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
                    } else {
                        logger.trace("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
                    }
                    PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
                    partitionResponseInfoList.add(partitionResponseInfo);
                } catch (MessageFormatException e) {
                    logger.error("Message format exception on a get with error code " + e.getErrorCode() + " for partitionRequestInfo " + partitionRequestInfo, e);
                    if (e.getErrorCode() == MessageFormatErrorCodes.Data_Corrupt) {
                        metrics.dataCorruptError.inc();
                    } else if (e.getErrorCode() == MessageFormatErrorCodes.Unknown_Format_Version) {
                        metrics.unknownFormatError.inc();
                    }
                    PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getMessageFormatErrorMapping(e.getErrorCode()));
                    partitionResponseInfoList.add(partitionResponseInfo);
                }
            }
        }
        CompositeSend compositeSend = new CompositeSend(messagesToSendList);
        response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, compositeSend, ServerErrorCode.No_Error);
    } catch (Exception e) {
        logger.error("Unknown exception for request " + getRequest, e);
        response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTime;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", getRequest, response, processingTime);
        if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
            metrics.getBlobProcessingTimeInMs.update(processingTime);
            metrics.updateGetBlobProcessingTimeBySize(response.sizeInBytes(), processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
            metrics.getBlobPropertiesProcessingTimeInMs.update(processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
            metrics.getBlobUserMetadataProcessingTimeInMs.update(processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
            metrics.getBlobInfoProcessingTimeInMs.update(processingTime);
        } else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
            metrics.getBlobAllProcessingTimeInMs.update(processingTime);
            metrics.updateGetBlobProcessingTimeBySize(response.sizeInBytes(), processingTime);
        }
    }
    sendGetResponse(requestResponseChannel, response, request, responseQueueTime, responseSendTime, responseTotalTime, totalTimeSpent, response.sizeInBytes(), getRequest.getMessageFormatFlag(), metrics);
}

Example 2 with PartitionResponseInfo

use of com.github.ambry.protocol.PartitionResponseInfo in project ambry by linkedin.

the class AmbryRequestsTest method sendAndVerifyOperationRequest.

/**
 * Sends and verifies that an operation specific request works correctly.
 * @param requestType the type of the request to send.
 * @param ids the partitionIds to send requests for.
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
 *                          response in the constituents rather than the actual response ({@link GetResponse} and
 *                          {@link ReplicaMetadataResponse}).
 * @throws InterruptedException
 * @throws IOException
 */
private void sendAndVerifyOperationRequest(RequestOrResponseType requestType, List<? extends PartitionId> ids, ServerErrorCode expectedErrorCode) throws InterruptedException, IOException {
    for (PartitionId id : ids) {
        int correlationId = TestUtils.RANDOM.nextInt();
        String clientId = UtilsTest.getRandomString(10);
        BlobId blobId = new BlobId(CommonTestUtils.getCurrentBlobIdVersion(), BlobId.BlobIdType.NATIVE, ClusterMapUtils.UNKNOWN_DATACENTER_ID, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), id, false);
        RequestOrResponse request;
        switch(requestType) {
            case PutRequest:
                BlobProperties properties = new BlobProperties(0, "serviceId", blobId.getAccountId(), blobId.getAccountId(), false);
                request = new PutRequest(correlationId, clientId, blobId, properties, ByteBuffer.allocate(0), ByteBuffer.allocate(0), 0, BlobType.DataBlob, null);
                break;
            case DeleteRequest:
                request = new DeleteRequest(correlationId, clientId, blobId, SystemTime.getInstance().milliseconds());
                break;
            case GetRequest:
                PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(id, Collections.singletonList(blobId));
                request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
                break;
            case ReplicaMetadataRequest:
                ReplicaMetadataRequestInfo rRequestInfo = new ReplicaMetadataRequestInfo(id, FIND_TOKEN_FACTORY.getNewFindToken(), "localhost", "/tmp");
                request = new ReplicaMetadataRequest(correlationId, clientId, Collections.singletonList(rRequestInfo), Long.MAX_VALUE);
                break;
            default:
                throw new IllegalArgumentException(requestType + " not supported by this function");
        }
        storageManager.resetStore();
        Response response = sendRequestGetResponse(request, requestType == RequestOrResponseType.GetRequest || requestType == RequestOrResponseType.ReplicaMetadataRequest ? ServerErrorCode.No_Error : expectedErrorCode);
        if (expectedErrorCode.equals(ServerErrorCode.No_Error)) {
            assertEquals("Operation received at the store not as expected", requestType, MockStorageManager.operationReceived);
        }
        if (requestType == RequestOrResponseType.GetRequest) {
            GetResponse getResponse = (GetResponse) response;
            for (PartitionResponseInfo info : getResponse.getPartitionResponseInfoList()) {
                assertEquals("Error code does not match expected", expectedErrorCode, info.getErrorCode());
            }
        } else if (requestType == RequestOrResponseType.ReplicaMetadataRequest) {
            ReplicaMetadataResponse replicaMetadataResponse = (ReplicaMetadataResponse) response;
            for (ReplicaMetadataResponseInfo info : replicaMetadataResponse.getReplicaMetadataResponseInfoList()) {
                assertEquals("Error code does not match expected", expectedErrorCode, info.getError());
            }
        }
    }
}

Example 3 with PartitionResponseInfo

use of com.github.ambry.protocol.PartitionResponseInfo in project ambry by linkedin.

the class AmbryServerRequestsTest method sendAndVerifyOperationRequest.

/**
 * Sends and verifies that an operation specific request works correctly.
 * @param request the {@link NetworkRequest} to send to {@link AmbryRequests}
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response. For some requests this is the
 *                          response in the constituents rather than the actual response ({@link GetResponse} and
 *                          {@link ReplicaMetadataResponse}).
 * @param forceCheckOpReceived if {@code true}, checks the operation received at the {@link Store} even if
 *                             there is an error expected. Always checks op received if {@code expectedErrorCode} is
 *                             {@link ServerErrorCode#No_Error}. Skips the check otherwise.
 * @throws InterruptedException
 * @throws IOException
 * @return the response associated with given request.
 */
private Response sendAndVerifyOperationRequest(RequestOrResponse request, ServerErrorCode expectedErrorCode, Boolean forceCheckOpReceived) throws InterruptedException, IOException {
    storageManager.resetStore();
    RequestOrResponseType requestType = request.getRequestType();
    Response response = sendRequestGetResponse(request, EnumSet.of(RequestOrResponseType.GetRequest, RequestOrResponseType.ReplicaMetadataRequest).contains(requestType) ? ServerErrorCode.No_Error : expectedErrorCode);
    if (expectedErrorCode.equals(ServerErrorCode.No_Error) || (forceCheckOpReceived && !expectedErrorCode.equals(ServerErrorCode.Temporarily_Disabled))) {
        assertEquals("Operation received at the store not as expected", requestType, MockStorageManager.operationReceived);
    }
    if (requestType == RequestOrResponseType.GetRequest) {
        GetResponse getResponse = (GetResponse) response;
        for (PartitionResponseInfo info : getResponse.getPartitionResponseInfoList()) {
            assertEquals("Error code does not match expected", expectedErrorCode, info.getErrorCode());
        }
    } else if (requestType == RequestOrResponseType.ReplicaMetadataRequest) {
        ReplicaMetadataResponse replicaMetadataResponse = (ReplicaMetadataResponse) response;
        for (ReplicaMetadataResponseInfo info : replicaMetadataResponse.getReplicaMetadataResponseInfoList()) {
            assertEquals("Error code does not match expected", expectedErrorCode, info.getError());
        }
    }
    return response;
}

Example 4 with PartitionResponseInfo

use of com.github.ambry.protocol.PartitionResponseInfo in project ambry by linkedin.

the class VcrRecoveryTest method testGetOnRecoveryNode.

/**
 * Do a get on recovery node to test that all the blobids that were uploaded to vcr node have been recovered on recovery node.
 * @param blobIdToSizeMap {@link Map} of blobid to size uploaded to vcr node.
 * @throws IOException on {@link IOException}
 */
private void testGetOnRecoveryNode(Map<BlobId, Integer> blobIdToSizeMap) throws IOException {
    ConnectedChannel channel = ServerTestUtil.getBlockingChannelBasedOnPortType(recoveryNodePort, "localhost", null, null);
    channel.connect();
    AtomicInteger correlationIdGenerator = new AtomicInteger(0);
    List<PartitionRequestInfo> partitionRequestInfoList = Collections.singletonList(new PartitionRequestInfo(partitionId, blobIds));
    GetRequest getRequest = new GetRequest(correlationIdGenerator.incrementAndGet(), GetRequest.Replication_Client_Id_Prefix + recoveryNode.getHostname(), MessageFormatFlags.All, partitionRequestInfoList, new ReplicationConfig(new VerifiableProperties(recoveryProperties)).replicationIncludeAll ? GetOption.Include_All : GetOption.None);
    channel.send(getRequest);
    GetResponse getResponse = GetResponse.readFrom(channel.receive().getInputStream(), recoveryCluster.getClusterMap());
    for (PartitionResponseInfo partitionResponseInfo : getResponse.getPartitionResponseInfoList()) {
        assertEquals("Error in getting the recovered blobs", ServerErrorCode.No_Error, partitionResponseInfo.getErrorCode());
        // old value is 272. Adding 8 Bytes due to the two fields added 4 + 4 Blob Property BYTE.
        for (MessageInfo messageInfo : partitionResponseInfo.getMessageInfoList()) {
            assertEquals(blobIdToSizeMap.get(messageInfo.getStoreKey()) + 280, messageInfo.getSize());
        }
    }
}

Example 5 with PartitionResponseInfo

use of com.github.ambry.protocol.PartitionResponseInfo in project ambry by linkedin.

the class AmbryServerRequestsTest method sendAndVerifyGetOriginalStoreKeys.

/**
 * Sends and verifies that GetRequest with a list of original blobIds works correctly.
 * @param blobIds List of blobIds for GetRequest.
 * @param expectedErrorCode the {@link ServerErrorCode} expected in the response.
 * @throws InterruptedException
 * @throws IOException
 */
private void sendAndVerifyGetOriginalStoreKeys(List<BlobId> blobIds, ServerErrorCode expectedErrorCode) throws InterruptedException, IOException {
    PartitionId partitionId = blobIds.get(0).getPartition();
    int correlationId = blobIds.get(0).getContainerId();
    String clientId = TestUtils.getRandomString(10);
    PartitionRequestInfo pRequestInfo = new PartitionRequestInfo(partitionId, blobIds);
    RequestOrResponse request = new GetRequest(correlationId, clientId, MessageFormatFlags.All, Collections.singletonList(pRequestInfo), GetOption.Include_All);
    storageManager.resetStore();
    if (!expectedErrorCode.equals(ServerErrorCode.Unknown_Error)) {
        // known error will be filled to each PartitionResponseInfo and set ServerErrorCode.No_Error in response.
        Response response = sendRequestGetResponse(request, ServerErrorCode.No_Error);
        assertEquals("Operation received at the store not as expected", RequestOrResponseType.GetRequest, MockStorageManager.operationReceived);
        for (PartitionResponseInfo info : ((GetResponse) response).getPartitionResponseInfoList()) {
            assertEquals("Error code does not match expected", expectedErrorCode, info.getErrorCode());
        }
        response.release();
    } else {
        sendRequestGetResponse(request, ServerErrorCode.Unknown_Error).release();
    }
}