Examples with RequestInfo com.github.ambry.network.RequestInfo used on opensource projects

Example 16 with RequestInfo

use of com.github.ambry.network.RequestInfo in project ambry by linkedin.

the class QuotaAwareOperationController method addToRequestQueue.

/**
 * Add the specified {@link List} of {@link RequestInfo}s to the request queue.
 * @param requestInfos {@link List} of {@link RequestInfo} objects.
 */
private void addToRequestQueue(List<RequestInfo> requestInfos) {
    for (RequestInfo requestInfo : requestInfos) {
        QuotaResource quotaResource = requestInfo.getChargeable().getQuotaResource();
        if (quotaResource == null) {
            quotaResource = UNKNOWN_QUOTA_RESOURCE;
        }
        getRequestQueue(requestInfo.getChargeable().getQuotaMethod()).putIfAbsent(quotaResource, new LinkedList<>());
        getRequestQueue(requestInfo.getChargeable().getQuotaMethod()).get(quotaResource).add(requestInfo);
    }
}

Example 17 with RequestInfo

use of com.github.ambry.network.RequestInfo in project ambry by linkedin.

the class NonBlockingRouterTestBase method testFailureDetectorNotification.

/**
 * Test that failure detector is correctly notified for all responses regardless of the order in which successful
 * and failed responses arrive.
 * @param opHelper the {@link OperationHelper}
 * @param networkClient the {@link SocketNetworkClient}
 * @param failedReplicaIds the list that will contain all the replicas for which failure was notified.
 * @param blobId the id of the blob to get/delete. For puts, this will be null.
 * @param successfulResponseCount the AtomicInteger that will contain the count of replicas for which success was
 *                                notified.
 * @param invalidResponse the AtomicBoolean that will contain whether an unexpected failure was notified.
 * @param indexToFail if greater than 0, the index representing which response for which failure is to be simulated.
 *                    For example, if index is 0, then the first response will be failed.
 *                    If the index is -1, no responses will be failed, and successful responses will be returned to
 *                    the operation managers.
 */
protected void testFailureDetectorNotification(OperationHelper opHelper, SocketNetworkClient networkClient, List<ReplicaId> failedReplicaIds, BlobId blobId, AtomicInteger successfulResponseCount, AtomicBoolean invalidResponse, int indexToFail) throws Exception {
    failedReplicaIds.clear();
    successfulResponseCount.set(0);
    invalidResponse.set(false);
    mockSelectorState.set(MockSelectorState.Good);
    FutureResult futureResult = opHelper.submitOperation(blobId);
    int requestParallelism = opHelper.requestParallelism;
    List<RequestInfo> allRequests = new ArrayList<>();
    Set<Integer> allDropped = new HashSet<>();
    long loopStartTimeMs = SystemTime.getInstance().milliseconds();
    while (allRequests.size() < requestParallelism) {
        if (loopStartTimeMs + AWAIT_TIMEOUT_MS < SystemTime.getInstance().milliseconds()) {
            Assert.fail("Waited too long for requests.");
        }
        opHelper.pollOpManager(allRequests, allDropped);
    }
    ReplicaId replicaIdToFail = indexToFail == -1 ? null : allRequests.get(indexToFail).getReplicaId();
    for (RequestInfo requestInfo : allRequests) {
        ResponseInfo responseInfo;
        if (replicaIdToFail != null && replicaIdToFail.equals(requestInfo.getReplicaId())) {
            responseInfo = new ResponseInfo(requestInfo, NetworkClientErrorCode.NetworkError, null);
            requestInfo.getRequest().release();
        } else {
            List<RequestInfo> requestInfoListToSend = new ArrayList<>();
            requestInfoListToSend.add(requestInfo);
            List<ResponseInfo> responseInfoList;
            loopStartTimeMs = SystemTime.getInstance().milliseconds();
            do {
                if (loopStartTimeMs + AWAIT_TIMEOUT_MS < SystemTime.getInstance().milliseconds()) {
                    Assert.fail("Waited too long for the response.");
                }
                responseInfoList = networkClient.sendAndPoll(requestInfoListToSend, Collections.emptySet(), 10);
                requestInfoListToSend.clear();
            } while (responseInfoList.size() == 0);
            responseInfo = responseInfoList.get(0);
        }
        opHelper.handleResponse(responseInfo);
        responseInfo.release();
    }
    // Poll once again so that the operation gets a chance to complete.
    allRequests.clear();
    if (testEncryption) {
        opHelper.awaitOpCompletionOrTimeOut(futureResult);
    } else {
        opHelper.pollOpManager(allRequests, allDropped);
    }
    futureResult.get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    Assert.assertEquals(0, allDropped.size());
    if (indexToFail == -1) {
        Assert.assertEquals("Successful notification should have arrived for replicas that were up", opHelper.requestParallelism, successfulResponseCount.get());
        Assert.assertEquals("Failure detector should not have been notified", 0, failedReplicaIds.size());
        Assert.assertFalse("There should be no notifications of any other kind", invalidResponse.get());
    } else {
        Assert.assertEquals("Failure detector should have been notified", 1, failedReplicaIds.size());
        Assert.assertEquals("Failed notification should have arrived for the failed replica", replicaIdToFail, failedReplicaIds.get(0));
        Assert.assertEquals("Successful notification should have arrived for replicas that were up", opHelper.requestParallelism - 1, successfulResponseCount.get());
        Assert.assertFalse("There should be no notifications of any other kind", invalidResponse.get());
    }
}

Example 18 with RequestInfo

use of com.github.ambry.network.RequestInfo in project ambry by linkedin.

the class MockRouterCallback method testNetworkClientTimeoutAllFailure.

/**
 * Test the case where all requests time out within the SocketNetworkClient.
 * @throws Exception
 */
@Test
public void testNetworkClientTimeoutAllFailure() throws Exception {
    NonBlockingRouter.currentOperationsCount.incrementAndGet();
    GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, null, routerCallback, kms, cryptoService, cryptoJobHandler, time, false, quotaChargeCallback);
    ArrayList<RequestInfo> requestListToFill = new ArrayList<>();
    requestRegistrationCallback.setRequestsToSend(requestListToFill);
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        for (RequestInfo requestInfo : requestListToFill) {
            ResponseInfo fakeResponse = new ResponseInfo(requestInfo, NetworkClientErrorCode.NetworkError, null);
            op.handleResponse(fakeResponse, null);
            fakeResponse.release();
            if (op.isOperationComplete()) {
                break;
            }
        }
        requestListToFill.clear();
    }
    // At this time requests would have been created for all replicas, as none of them were delivered,
    // and cross-colo proxying is enabled by default.
    Assert.assertEquals("Must have attempted sending requests to all replicas", replicasCount, correlationIdToGetOperation.size());
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    RouterException routerException = (RouterException) op.getOperationException();
    Assert.assertEquals(RouterErrorCode.OperationTimedOut, routerException.getErrorCode());
}

Example 19 with RequestInfo

use of com.github.ambry.network.RequestInfo in project ambry by linkedin.

the class MockRouterCallback method testPollAndResponseHandling.

/**
 * Test basic successful operation completion, by polling and handing over responses to the BlobInfo operation.
 * @throws Exception
 */
@Test
public void testPollAndResponseHandling() throws Exception {
    for (short expectedLifeVersion : new short[] { 0, 1 }) {
        NonBlockingRouter.currentOperationsCount.incrementAndGet();
        // Now set the lifeVersion
        for (MockServer mockServer : mockServerLayout.getMockServers()) {
            if (mockServer.getBlobs().containsKey(blobId.getID())) {
                mockServer.getBlobs().get(blobId.getID()).lifeVersion = expectedLifeVersion;
            }
        }
        GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, null, routerCallback, kms, cryptoService, cryptoJobHandler, time, false, quotaChargeCallback);
        ArrayList<RequestInfo> requestListToFill = new ArrayList<>();
        requestRegistrationCallback.setRequestsToSend(requestListToFill);
        op.poll(requestRegistrationCallback);
        Assert.assertEquals("There should only be as many requests at this point as requestParallelism", requestParallelism, correlationIdToGetOperation.size());
        CountDownLatch onPollLatch = new CountDownLatch(1);
        if (testEncryption) {
            routerCallback.setOnPollLatch(onPollLatch);
        }
        List<ResponseInfo> responses = sendAndWaitForResponses(requestListToFill);
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
            op.handleResponse(responseInfo, getResponse);
            if (op.isOperationComplete()) {
                break;
            }
        }
        responses.forEach(ResponseInfo::release);
        if (testEncryption) {
            Assert.assertTrue("Latch should have been zeroed ", onPollLatch.await(500, TimeUnit.MILLISECONDS));
            op.poll(requestRegistrationCallback);
        }
        Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
        assertSuccess(op, expectedLifeVersion);
        // poll again to make sure that counters aren't triggered again (check in @After)
        op.poll(requestRegistrationCallback);
        correlationIdToGetOperation.clear();
    }
}

Example 20 with RequestInfo

use of com.github.ambry.network.RequestInfo in project ambry by linkedin.

the class PutOperationTest method testSendIncomplete.

/**
 * Ensure that if any of the requests associated with the buffer of a PutChunk is not completely read out even
 * after the associated chunk is complete, the buffer is not reused even though the PutChunk is reused.
 */
@Test
public void testSendIncomplete() throws Exception {
    int numChunks = routerConfig.routerMaxInMemPutChunks + 1;
    BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
    byte[] userMetadata = new byte[10];
    byte[] content = new byte[chunkSize * numChunks];
    random.nextBytes(content);
    ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
    FutureResult<String> future = new FutureResult<>();
    MockNetworkClient mockNetworkClient = new MockNetworkClient();
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, channel, PutBlobOptions.DEFAULT, future, null, new RouterCallback(mockNetworkClient, new ArrayList<>()), null, null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    List<RequestInfo> requestInfos = new ArrayList<>();
    requestRegistrationCallback.setRequestsToSend(requestInfos);
    // Since this channel is in memory, one call to fill chunks would end up filling the maximum number of PutChunks.
    op.fillChunks();
    Assert.assertTrue("ReadyForPollCallback should have been invoked as chunks were fully filled", mockNetworkClient.getAndClearWokenUpStatus());
    // A poll should therefore return requestParallelism number of requests from each chunk
    op.poll(requestRegistrationCallback);
    Assert.assertEquals(routerConfig.routerMaxInMemPutChunks * requestParallelism, requestInfos.size());
    // There are routerMaxInMemPutChunks + 1 data chunks for this blob (and a metadata chunk).
    // Once the first chunk is completely sent out, the first PutChunk will be reused. What the test verifies is that
    // the buffer of the first PutChunk does not get reused. It does this as follows:
    // For the first chunk,
    // 1. use first request to succeed the chunk (the successTarget is set to 1).
    // 2. read and store from the second for comparing later.
    // 3. read from the third after the first PutChunk gets reused and ensure that the data from the third is the
    // same as from what was saved off from the second. This means that the buffer was not reused by the first
    // PutChunk.
    // 1.
    ResponseInfo responseInfo = getResponseInfo(requestInfos.get(0));
    PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
    op.handleResponse(responseInfo, putResponse);
    requestInfos.get(0).getRequest().release();
    responseInfo.release();
    // 2.
    PutRequest putRequest = (PutRequest) requestInfos.get(1).getRequest();
    ByteBuffer buf = ByteBuffer.allocate((int) putRequest.sizeInBytes());
    ByteBufferChannel bufChannel = new ByteBufferChannel(buf);
    // read it out (which also marks this request as complete).
    putRequest.writeTo(bufChannel);
    putRequest.release();
    byte[] expectedRequestContent = buf.array();
    // 3.
    // first save the third request
    PutRequest savedRequest = (PutRequest) requestInfos.get(2).getRequest();
    // succeed all the other requests.
    for (int i = 3; i < requestInfos.size(); i++) {
        responseInfo = getResponseInfo(requestInfos.get(i));
        putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
        op.handleResponse(responseInfo, putResponse);
        requestInfos.get(i).getRequest().release();
        responseInfo.release();
    }
    // fill the first PutChunk with the last chunk.
    op.fillChunks();
    // Verify that the last chunk was filled.
    requestInfos.clear();
    op.poll(requestRegistrationCallback);
    Assert.assertEquals(1 * requestParallelism, requestInfos.size());
    // Verify that the buffer of the third request is not affected.
    buf = ByteBuffer.allocate((int) savedRequest.sizeInBytes());
    bufChannel = new ByteBufferChannel(buf);
    savedRequest.writeTo(bufChannel);
    savedRequest.release();
    byte[] savedRequestContent = buf.array();
    // reset the correlation id as they will be different between the two requests.
    resetCorrelationId(expectedRequestContent);
    resetCorrelationId(savedRequestContent);
    Assert.assertArrayEquals("Underlying buffer should not have be reused", expectedRequestContent, savedRequestContent);
    // internal to the chunk (though this can be verified via coverage).
    for (int i = 0; i < requestInfos.size(); i++) {
        responseInfo = getResponseInfo(requestInfos.get(i));
        putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
        op.handleResponse(responseInfo, putResponse);
        requestInfos.get(i).getRequest().release();
        responseInfo.release();
    }
    requestInfos.clear();
    // this should return requests for the metadata chunk
    op.poll(requestRegistrationCallback);
    Assert.assertEquals(1 * requestParallelism, requestInfos.size());
    Assert.assertFalse("Operation should not be complete yet", op.isOperationComplete());
    // once the metadata request succeeds, it should complete the operation.
    responseInfo = getResponseInfo(requestInfos.get(0));
    putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
    op.handleResponse(responseInfo, putResponse);
    responseInfo.release();
    requestInfos.forEach(info -> info.getRequest().release());
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
}