Examples with ResponseInfo com.github.ambry.network.ResponseInfo used on opensource projects

Example 31 with ResponseInfo

use of com.github.ambry.network.ResponseInfo in project ambry by linkedin.

the class BackgroundDeleter method onResponse.

/**
 * Handle the response from polling the {@link NetworkClient}.
 * @param responseInfoList the list of {@link ResponseInfo} containing the responses.
 */
protected void onResponse(List<ResponseInfo> responseInfoList) {
    for (ResponseInfo responseInfo : responseInfoList) {
        try {
            RequestInfo requestInfo = responseInfo.getRequestInfo();
            if (requestInfo == null) {
                // If requestInfo is null, it means request has been failed previously due to long wait in pending requests
                // queue. The failed request was already handled by one of the managers(PutManager, GetManager, etc). Current
                // response comes from timed-out connection associated with previous request. Router only needs to notify
                // responseHandler to mark the data node resource down.
                DataNodeId dataNodeId = responseInfo.getDataNode();
                responseHandler.onConnectionTimeout(dataNodeId);
            } else {
                long responseReceiveTime = requestInfo.getStreamHeaderFrameReceiveTime();
                if (responseReceiveTime != -1) {
                    routerMetrics.responseReceiveToHandleLatencyMs.update(System.currentTimeMillis() - responseReceiveTime);
                }
                RequestOrResponseType type = ((RequestOrResponse) requestInfo.getRequest()).getRequestType();
                logger.debug("Handling response of type {} for {}", type, requestInfo.getRequest().getCorrelationId());
                switch(type) {
                    case PutRequest:
                        putManager.handleResponse(responseInfo);
                        break;
                    case GetRequest:
                        getManager.handleResponse(responseInfo);
                        break;
                    case DeleteRequest:
                        deleteManager.handleResponse(responseInfo);
                        break;
                    case TtlUpdateRequest:
                        ttlUpdateManager.handleResponse(responseInfo);
                        break;
                    case UndeleteRequest:
                        undeleteManager.handleResponse(responseInfo);
                        break;
                    default:
                        logger.error("Unexpected response type: {} received, discarding", type);
                }
            }
        } catch (Exception e) {
            logger.error("Unexpected error received while handling a response: ", e);
            routerMetrics.operationManagerHandleResponseErrorCount.inc();
        }
    }
}

Example 32 with ResponseInfo

use of com.github.ambry.network.ResponseInfo in project ambry by linkedin.

the class CloudOperationTest method doDirectPut.

/**
 * Do a put directly to the mock servers. This allows for blobs with malformed properties to be constructed.
 * @param blobProperties the {@link BlobProperties} for the blob.
 * @param userMetadata user meta data of the blob.
 * @param blobContent the raw content for the blob to upload (i.e. this can be serialized composite blob metadata or
 *                    an encrypted blob).
 * @return the blob id
 * @throws Exception Any unexpected exception
 */
private BlobId doDirectPut(BlobProperties blobProperties, byte[] userMetadata, ByteBuf blobContent) throws Exception {
    List<PartitionId> writablePartitionIds = mockClusterMap.getWritablePartitionIds(MockClusterMap.DEFAULT_PARTITION_CLASS);
    PartitionId partitionId = writablePartitionIds.get(random.nextInt(writablePartitionIds.size()));
    BlobId blobId = new BlobId(routerConfig.routerBlobidCurrentVersion, BlobId.BlobIdType.NATIVE, mockClusterMap.getLocalDatacenterId(), blobProperties.getAccountId(), blobProperties.getContainerId(), partitionId, blobProperties.isEncrypted(), BlobId.BlobDataType.DATACHUNK);
    Iterator<MockServer> servers = partitionId.getReplicaIds().stream().map(ReplicaId::getDataNodeId).map(dataNodeId -> mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort())).iterator();
    ByteBuffer userMetadataBuf = ByteBuffer.wrap(userMetadata);
    while (servers.hasNext()) {
        MockServer server = servers.next();
        PutRequest request = new PutRequest(random.nextInt(), "clientId", blobId, blobProperties, userMetadataBuf.duplicate(), blobContent.retainedDuplicate(), blobContent.readableBytes(), BlobType.DataBlob, null);
        // Make sure we release the BoundedNettyByteBufReceive.
        server.send(request).release();
        request.release();
    }
    // send to Cloud destinations.
    PutRequest request = new PutRequest(random.nextInt(), "clientId", blobId, blobProperties, userMetadataBuf.duplicate(), blobContent.retainedDuplicate(), blobContent.readableBytes(), BlobType.DataBlob, null);
    // Get the cloud replica.
    ReplicaId replica = partitionId.getReplicaIds().get(0);
    Assert.assertEquals("It should be a cloud backed replica.", replica.getReplicaType(), ReplicaType.CLOUD_BACKED);
    String hostname = replica.getDataNodeId().getHostname();
    Port port = new Port(-1, PortType.PLAINTEXT);
    List<RequestInfo> requestList = new ArrayList<>();
    RequestInfo requestInfo = new RequestInfo(hostname, port, request, replica, null);
    requestList.add(requestInfo);
    List<ResponseInfo> responseList = sendAndWaitForResponses(requestList);
    request.release();
    blobContent.release();
    return blobId;
}

Example 33 with ResponseInfo

use of com.github.ambry.network.ResponseInfo in project ambry by linkedin.

the class CloudOperationTest method getBlobAndAssertSuccess.

/**
 * Construct GetBlob operations with appropriate callbacks, then poll those operations until they complete,
 * and ensure that the whole blob data is read out and the contents match.
 * @param blobId id of the blob to get
 * @param expectedLifeVersion the expected lifeVersion from get operation.
 * @param expectedBlobSize the expected blob size
 * @param expectedBlobProperties  the expected {@link BlobProperties} for the blob.
 * @param expectedUserMetadata the expected user meta data
 * @param expectPutContent the expected blob content
 * @param options options of the get blob operation
 * @throws Exception Any unexpected exception
 */
private void getBlobAndAssertSuccess(final BlobId blobId, final short expectedLifeVersion, final int expectedBlobSize, final BlobProperties expectedBlobProperties, final byte[] expectedUserMetadata, final byte[] expectPutContent, final GetBlobOptionsInternal options) throws Exception {
    final CountDownLatch readCompleteLatch = new CountDownLatch(1);
    final AtomicLong readCompleteResult = new AtomicLong(0);
    // callback to compare the data
    Callback<GetBlobResultInternal> callback = (result, exception) -> {
        Assert.assertNull("Shouldn't have exception", exception);
        try {
            BlobInfo blobInfo;
            switch(options.getBlobOptions.getOperationType()) {
                case All:
                    Assert.assertFalse("not supposed to be raw mode", options.getBlobOptions.isRawMode());
                    blobInfo = result.getBlobResult.getBlobInfo();
                    Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(expectedBlobProperties, blobInfo.getBlobProperties()));
                    Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", expectedBlobSize, blobInfo.getBlobProperties().getBlobSize());
                    Assert.assertArrayEquals("User metadata must be the same", expectedUserMetadata, blobInfo.getUserMetadata());
                    Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
                    break;
                case Data:
                    Assert.assertNull("Unexpected blob info in operation result", result.getBlobResult.getBlobInfo());
                    break;
                case BlobInfo:
                    blobInfo = result.getBlobResult.getBlobInfo();
                    Assert.assertTrue("Blob properties must be the same", RouterTestHelpers.arePersistedFieldsEquivalent(expectedBlobProperties, blobInfo.getBlobProperties()));
                    Assert.assertEquals("Blob size should in received blobProperties should be the same as actual", expectedBlobSize, blobInfo.getBlobProperties().getBlobSize());
                    Assert.assertNull("Unexpected blob data in operation result", result.getBlobResult.getBlobDataChannel());
                    Assert.assertEquals("LifeVersion mismatch", expectedLifeVersion, blobInfo.getLifeVersion());
            }
        } catch (Throwable e) {
            Assert.fail("Shouldn't receive exception here");
        }
        if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo) {
            final ByteBufferAsyncWritableChannel asyncWritableChannel = new ByteBufferAsyncWritableChannel();
            Utils.newThread(() -> {
                Future<Long> readIntoFuture = result.getBlobResult.getBlobDataChannel().readInto(asyncWritableChannel, null);
                assertBlobReadSuccess(options.getBlobOptions, readIntoFuture, asyncWritableChannel, result.getBlobResult.getBlobDataChannel(), readCompleteLatch, readCompleteResult, expectedBlobSize, expectPutContent);
            }, false).start();
        } else {
            readCompleteLatch.countDown();
        }
    };
    // create GetBlobOperation
    final Map<Integer, GetOperation> correlationIdToGetOperation = new HashMap<>();
    final RequestRegistrationCallback<GetOperation> requestRegistrationCallback = new RequestRegistrationCallback<>(correlationIdToGetOperation);
    NonBlockingRouter.currentOperationsCount.incrementAndGet();
    GetBlobOperation op = new GetBlobOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, callback, routerCallback, blobIdFactory, null, null, null, time, false, null);
    requestRegistrationCallback.setRequestsToSend(new ArrayList<>());
    // Wait operation to complete
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = RouterUtils.extractResponseAndNotifyResponseHandler(responseHandler, routerMetrics, responseInfo, stream -> GetResponse.readFrom(stream, mockClusterMap), response -> {
                ServerErrorCode serverError = response.getError();
                if (serverError == ServerErrorCode.No_Error) {
                    serverError = response.getPartitionResponseInfoList().get(0).getErrorCode();
                }
                return serverError;
            });
            op.handleResponse(responseInfo, getResponse);
            responseInfo.release();
        }
    }
    readCompleteLatch.await();
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    // Ensure that a ChannelClosed exception is not set when the ReadableStreamChannel is closed correctly.
    Assert.assertNull("Callback operation exception should be null", op.getOperationException());
    if (options.getBlobOptions.getOperationType() != GetBlobOptions.OperationType.BlobInfo && !options.getBlobOptions.isRawMode() && !options.getChunkIdsOnly) {
        int sizeWritten = expectedBlobSize;
        if (options.getBlobOptions.getRange() != null) {
            ByteRange range = options.getBlobOptions.getRange().toResolvedByteRange(expectedBlobSize, options.getBlobOptions.resolveRangeOnEmptyBlob());
            sizeWritten = (int) range.getRangeSize();
        }
        Assert.assertEquals("Size read must equal size written", sizeWritten, readCompleteResult.get());
    }
}

Example 34 with ResponseInfo

use of com.github.ambry.network.ResponseInfo in project ambry by linkedin.

the class GetBlobOperationTest method testNetworkClientTimeoutAllFailure.

/**
 * Test the case where all requests time out within the SocketNetworkClient.
 * @throws Exception
 */
@Test
public void testNetworkClientTimeoutAllFailure() throws Exception {
    doPut();
    GetBlobOperation op = createOperation(routerConfig, null);
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        for (RequestInfo requestInfo : requestRegistrationCallback.getRequestsToSend()) {
            ResponseInfo fakeResponse = new ResponseInfo(requestInfo, NetworkClientErrorCode.NetworkError, null);
            op.handleResponse(fakeResponse, null);
            fakeResponse.release();
            if (op.isOperationComplete()) {
                break;
            }
        }
        requestRegistrationCallback.getRequestsToSend().clear();
    }
    // At this time requests would have been created for all replicas, as none of them were delivered,
    // and cross-colo proxying is enabled by default.
    Assert.assertEquals("Must have attempted sending requests to all replicas", replicasCount, correlationIdToGetOperation.size());
    assertFailureAndCheckErrorCode(op, RouterErrorCode.OperationTimedOut);
}

Example 35 with ResponseInfo

use of com.github.ambry.network.ResponseInfo in project ambry by linkedin.

the class GetBlobOperationTest method testTimeoutAndBlobNotFoundInOriginDc.

/**
 * Test the case where origin replicas return Blob_Not_found and the rest times out.
 * @throws Exception
 */
@Test
public void testTimeoutAndBlobNotFoundInOriginDc() throws Exception {
    assumeTrue(operationTrackerType.equals(AdaptiveOperationTracker.class.getSimpleName()));
    doPut();
    // Pick a remote DC as the new local DC.
    String newLocal = "DC1";
    String oldLocal = localDcName;
    Properties props = getDefaultNonBlockingRouterProperties(true);
    props.setProperty("router.datacenter.name", newLocal);
    props.setProperty("router.get.request.parallelism", "3");
    props.setProperty("router.operation.tracker.max.inflight.requests", "3");
    routerConfig = new RouterConfig(new VerifiableProperties(props));
    GetBlobOperation op = createOperation(routerConfig, null);
    correlationIdToGetOperation.clear();
    for (MockServer server : mockServerLayout.getMockServers()) {
        server.setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
    }
    op.poll(requestRegistrationCallback);
    time.sleep(routerConfig.routerRequestTimeoutMs + 1);
    // The request should have response from all remote replicas.
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
            op.handleResponse(responseInfo, getResponse);
            responseInfo.release();
        }
    }
    RouterException routerException = (RouterException) op.getOperationException();
    // error code should be OperationTimedOut because it precedes BlobDoesNotExist
    Assert.assertEquals(RouterErrorCode.BlobDoesNotExist, routerException.getErrorCode());
    props = getDefaultNonBlockingRouterProperties(true);
    props.setProperty("router.datacenter.name", oldLocal);
    props.setProperty("router.get.request.parallelism", "2");
    props.setProperty("router.operation.tracker.max.inflight.requests", "2");
    routerConfig = new RouterConfig(new VerifiableProperties(props));
}