Examples with ByteBufferReadableStreamChannel com.github.ambry.commons.ByteBufferReadableStreamChannel used on opensource projects

Example 21 with ByteBufferReadableStreamChannel

use of com.github.ambry.commons.ByteBufferReadableStreamChannel in project ambry by linkedin.

the class PutOperationTest method testSlippedPutsWithServerErrors.

/**
 * Test PUT operation that handles ServerErrorCode = Temporarily_Disabled and Replica_Unavailable
 * @throws Exception
 */
@Test
public void testSlippedPutsWithServerErrors() throws Exception {
    Properties properties = new Properties();
    properties.setProperty("router.hostname", "localhost");
    properties.setProperty("router.datacenter.name", "DC1");
    properties.setProperty("router.max.put.chunk.size.bytes", Integer.toString(chunkSize));
    properties.setProperty("router.put.request.parallelism", Integer.toString(requestParallelism));
    // Expect at least two successes so that you can create slipped puts.
    properties.setProperty("router.put.success.target", Integer.toString(2));
    VerifiableProperties vProps = new VerifiableProperties(properties);
    RouterConfig routerConfig = new RouterConfig(vProps);
    int numChunks = 1;
    BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
    byte[] userMetadata = new byte[10];
    byte[] content = new byte[chunkSize * numChunks];
    random.nextBytes(content);
    ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
    MockNetworkClient mockNetworkClient = new MockNetworkClient();
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, channel, PutBlobOptions.DEFAULT, new FutureResult<>(), null, new RouterCallback(mockNetworkClient, new ArrayList<>()), null, null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    List<RequestInfo> requestInfos = new ArrayList<>();
    requestRegistrationCallback.setRequestsToSend(requestInfos);
    // fill chunks would end up filling the maximum number of PutChunks.
    op.fillChunks();
    Assert.assertTrue("ReadyForPollCallback should have been invoked as chunks were fully filled", mockNetworkClient.getAndClearWokenUpStatus());
    // poll to populate request
    op.poll(requestRegistrationCallback);
    // Set up server errors such that put fails on 2 out 3 nodes, hence creating a slipped put on the succeeding node.
    // Second attempts on all node succeed.
    List<ServerErrorCode> serverErrorList = new ArrayList<>();
    // Success on the first host, slipped put
    serverErrorList.add(ServerErrorCode.No_Error);
    // Fail on the second host
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    // Fail on the third host
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    // Success on the second attempts on all hosts
    serverErrorList.add(ServerErrorCode.No_Error);
    serverErrorList.add(ServerErrorCode.No_Error);
    serverErrorList.add(ServerErrorCode.No_Error);
    mockServer.setServerErrors(serverErrorList);
    // Send all requests.
    for (int i = 0; i < requestInfos.size(); i++) {
        ResponseInfo responseInfo = getResponseInfo(requestInfos.get(i));
        PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
        op.handleResponse(responseInfo, putResponse);
        requestInfos.get(i).getRequest().release();
        responseInfo.release();
    }
    Assert.assertEquals("Number of slipped puts should be 1", 1, op.getSlippedPutBlobIds().size());
    // fill chunks again.
    op.fillChunks();
    requestInfos.clear();
    // poll to populate request
    op.poll(requestRegistrationCallback);
    // Send all requests again.
    for (int i = 0; i < requestInfos.size(); i++) {
        ResponseInfo responseInfo = getResponseInfo(requestInfos.get(i));
        PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
        op.handleResponse(responseInfo, putResponse);
        requestInfos.get(i).getRequest().release();
        responseInfo.release();
    }
    Assert.assertEquals("Number of slipped puts should be 1", 1, op.getSlippedPutBlobIds().size());
    PutOperation.PutChunk putChunk = op.getPutChunks().get(0);
    // Make sure the chunk blob id which has been put successfully is not part of the slipped puts.
    Assert.assertFalse(op.getSlippedPutBlobIds().contains(putChunk.chunkBlobId));
}

Example 22 with ByteBufferReadableStreamChannel

use of com.github.ambry.commons.ByteBufferReadableStreamChannel in project ambry by linkedin.

the class PutOperationTest method testHandleResponseWithServerErrors.

/**
 * Test PUT operation that handles ServerErrorCode = Temporarily_Disabled and Replica_Unavailable
 * @throws Exception
 */
@Test
public void testHandleResponseWithServerErrors() throws Exception {
    int numChunks = routerConfig.routerMaxInMemPutChunks + 1;
    BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
    byte[] userMetadata = new byte[10];
    byte[] content = new byte[chunkSize * numChunks];
    random.nextBytes(content);
    ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, channel, PutBlobOptions.DEFAULT, new FutureResult<>(), null, new RouterCallback(new MockNetworkClient(), new ArrayList<>()), null, null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    List<RequestInfo> requestInfos = new ArrayList<>();
    requestRegistrationCallback.setRequestsToSend(requestInfos);
    // fill chunks would end up filling the maximum number of PutChunks.
    op.fillChunks();
    // poll to populate request
    op.poll(requestRegistrationCallback);
    // make 1st request of first chunk encounter Temporarily_Disabled
    mockServer.setServerErrorForAllRequests(ServerErrorCode.Temporarily_Disabled);
    ResponseInfo responseInfo = getResponseInfo(requestInfos.get(0));
    PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
    op.handleResponse(responseInfo, putResponse);
    responseInfo.release();
    PutOperation.PutChunk putChunk = op.getPutChunks().get(0);
    SimpleOperationTracker operationTracker = (SimpleOperationTracker) putChunk.getOperationTrackerInUse();
    Assert.assertEquals("Disabled count should be 1", 1, operationTracker.getDisabledCount());
    Assert.assertEquals("Disabled count should be 0", 0, operationTracker.getFailedCount());
    // make 2nd request of first chunk encounter Replica_Unavailable
    mockServer.setServerErrorForAllRequests(ServerErrorCode.Replica_Unavailable);
    responseInfo = getResponseInfo(requestInfos.get(1));
    putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
    op.handleResponse(responseInfo, putResponse);
    responseInfo.release();
    putChunk = op.getPutChunks().get(0);
    Assert.assertEquals("Failure count should be 1", 1, ((SimpleOperationTracker) putChunk.getOperationTrackerInUse()).getFailedCount());
    mockServer.resetServerErrors();
    // Release all the other requests
    requestInfos.forEach(info -> info.getRequest().release());
}

Example 23 with ByteBufferReadableStreamChannel

use of com.github.ambry.commons.ByteBufferReadableStreamChannel in project ambry by linkedin.

the class UndeleteManagerTest method setup.

@Before
public void setup() throws Exception {
    blobIds.clear();
    for (int i = 0; i < BLOBS_COUNT; i++) {
        ReadableStreamChannel putChannel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(PUT_CONTENT));
        BlobProperties putBlobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
        String blobId = router.putBlob(putBlobProperties, new byte[0], putChannel, new PutBlobOptionsBuilder().build()).get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        blobIds.add(blobId);
        // Make sure all the mock servers have this put
        BlobId id = new BlobId(blobId, clusterMap);
        for (MockServer server : serverLayout.getMockServers()) {
            if (!server.getBlobs().containsKey(blobId)) {
                server.send(new PutRequest(NonBlockingRouter.correlationIdGenerator.incrementAndGet(), routerConfig.routerHostname, id, putBlobProperties, ByteBuffer.wrap(new byte[0]), Unpooled.wrappedBuffer(PUT_CONTENT), PUT_CONTENT.length, BlobType.DataBlob, null)).release();
            }
        }
    }
    undeleteManager = new UndeleteManager(clusterMap, new ResponseHandler(clusterMap), new LoggingNotificationSystem(), accountService, routerConfig, metrics, time);
    networkClient = networkClientFactory.getNetworkClient();
}

Example 24 with ByteBufferReadableStreamChannel

use of com.github.ambry.commons.ByteBufferReadableStreamChannel in project ambry by linkedin.

the class RouterServerTestFramework method startPutBlob.

/**
 * Submit a putBlob operation.
 * @param opChain the {@link OperationChain} object that this operation is a part of.
 */
private void startPutBlob(OperationChain opChain) {
    ReadableStreamChannel putChannel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(opChain.data));
    Callback<String> callback = new TestCallback<String>(opChain, false) {

        @Override
        void action(String result) {
            opChain.blobId = result;
        }
    };
    Future<String> future = router.putBlob(opChain.properties, opChain.userMetadata, putChannel, new PutBlobOptionsBuilder().build(), callback, quotaChargeCallback);
    TestFuture<String> testFuture = new TestFuture<String>(future, genLabel("putBlob", false), opChain) {

        @Override
        void check() throws Exception {
            checkBlobId(get(), opChain.properties, getOperationName());
        }
    };
    opChain.testFutures.add(testFuture);
}

Example 25 with ByteBufferReadableStreamChannel

use of com.github.ambry.commons.ByteBufferReadableStreamChannel in project ambry by linkedin.

the class NonBlockingRouterTest method testBadCallbackForUpdateTtl.

/**
 * Test that a bad user defined callback will not crash the router or the manager.
 * @throws Exception
 */
@Test
public void testBadCallbackForUpdateTtl() throws Exception {
    try {
        MockServerLayout serverLayout = new MockServerLayout(mockClusterMap);
        setRouter(getNonBlockingRouterProperties("DC1"), serverLayout, new LoggingNotificationSystem());
        setOperationParams();
        String blobId = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        putChannel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(putContent));
        String blobIdCheck = router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        testWithErrorCodes(Collections.singletonMap(ServerErrorCode.No_Error, 9), serverLayout, null, expectedError -> {
            final CountDownLatch callbackCalled = new CountDownLatch(1);
            router.updateBlobTtl(blobId, null, Utils.Infinite_Time, (result, exception) -> {
                callbackCalled.countDown();
                throw new RuntimeException("Throwing an exception in the user callback");
            }, null).get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
            assertTrue("Callback not called.", callbackCalled.await(10, TimeUnit.MILLISECONDS));
            assertEquals("All operations should be finished.", 0, router.getOperationsCount());
            assertTrue("Router should not be closed", router.isOpen());
            assertTtl(router, Collections.singleton(blobId), Utils.Infinite_Time);
            // Test that TtlUpdateManager is still functional
            router.updateBlobTtl(blobIdCheck, null, Utils.Infinite_Time).get(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
            assertTtl(router, Collections.singleton(blobIdCheck), Utils.Infinite_Time);
        });
    } finally {
        if (router != null) {
            router.close();
        }
    }
}