Examples with NettyByteBufDataInputStream com.github.ambry.utils.NettyByteBufDataInputStream used on opensource projects

Example 11 with NettyByteBufDataInputStream

use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.

the class ReplicaThread method fixMissingStoreKeys.

/**
 * Gets all the messages from the remote node for the missing keys and writes them to the local store
 * @param connectedChannel The connected channel that represents a connection to the remote replica
 * @param replicasToReplicatePerNode The information about the replicas that is being replicated
 * @param exchangeMetadataResponseList The missing keys in the local stores whose message needs to be retrieved
 *                                     from the remote stores
 * @param remoteColoGetRequestForStandby boolean which indicates if we are getting missing keys for standby or
 *                                       non-leader replica pairs during leader-based replication.
 * @throws IOException
 * @throws ReplicationException
 */
void fixMissingStoreKeys(ConnectedChannel connectedChannel, List<RemoteReplicaInfo> replicasToReplicatePerNode, List<ExchangeMetadataResponse> exchangeMetadataResponseList, boolean remoteColoGetRequestForStandby) throws IOException, ReplicationException {
    long fixMissingStoreKeysStartTimeInMs = time.milliseconds();
    GetResponse getResponse = null;
    try {
        if (exchangeMetadataResponseList.size() != replicasToReplicatePerNode.size() || replicasToReplicatePerNode.size() == 0) {
            throw new IllegalArgumentException("ExchangeMetadataResponseList size " + exchangeMetadataResponseList.size() + " and replicasToReplicatePerNode size " + replicasToReplicatePerNode.size() + " should be the same and greater than zero");
        }
        DataNodeId remoteNode = replicasToReplicatePerNode.get(0).getReplicaId().getDataNodeId();
        getResponse = getMessagesForMissingKeys(connectedChannel, exchangeMetadataResponseList, replicasToReplicatePerNode, remoteNode, remoteColoGetRequestForStandby);
        writeMessagesToLocalStoreAndAdvanceTokens(exchangeMetadataResponseList, getResponse, replicasToReplicatePerNode, remoteNode, remoteColoGetRequestForStandby);
    } finally {
        if (getResponse != null && getResponse.getInputStream() instanceof NettyByteBufDataInputStream) {
            // if the InputStream is NettyByteBufDataInputStream based, it's time to release its buffer.
            ((NettyByteBufDataInputStream) (getResponse.getInputStream())).getBuffer().release();
        }
        long fixMissingStoreKeysTime = time.milliseconds() - fixMissingStoreKeysStartTimeInMs;
        replicationMetrics.updateFixMissingStoreKeysTime(fixMissingStoreKeysTime, replicatingFromRemoteColo, replicatingOverSsl, datacenterName);
    }
}

Example 12 with NettyByteBufDataInputStream

use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.

the class ReplicaThread method getReplicaMetadataResponse.

/**
 * Gets the replica metadata response for a list of remote replicas on a given remote data node
 * @param replicasToReplicatePerNode The list of remote replicas for a node
 * @param connectedChannel The connection channel to the node
 * @param remoteNode The remote node from which replication needs to happen
 * @return ReplicaMetadataResponse, the response from replica metadata request to remote node
 * @throws ReplicationException
 * @throws IOException
 */
ReplicaMetadataResponse getReplicaMetadataResponse(List<RemoteReplicaInfo> replicasToReplicatePerNode, ConnectedChannel connectedChannel, DataNodeId remoteNode) throws ReplicationException, IOException {
    long replicaMetadataRequestStartTime = time.milliseconds();
    List<ReplicaMetadataRequestInfo> replicaMetadataRequestInfoList = new ArrayList<ReplicaMetadataRequestInfo>();
    for (RemoteReplicaInfo remoteReplicaInfo : replicasToReplicatePerNode) {
        ReplicaMetadataRequestInfo replicaMetadataRequestInfo = new ReplicaMetadataRequestInfo(remoteReplicaInfo.getReplicaId().getPartitionId(), remoteReplicaInfo.getToken(), dataNodeId.getHostname(), remoteReplicaInfo.getLocalReplicaId().getReplicaPath(), remoteReplicaInfo.getReplicaId().getReplicaType(), replicationConfig.replicaMetadataRequestVersion);
        replicaMetadataRequestInfoList.add(replicaMetadataRequestInfo);
        logger.trace("Remote node: {} Thread name: {} Remote replica: {} Token going to be sent to remote: {} ", remoteNode, threadName, remoteReplicaInfo.getReplicaId(), remoteReplicaInfo.getToken());
    }
    ChannelOutput channelOutput = null;
    try {
        ReplicaMetadataRequest request = new ReplicaMetadataRequest(correlationIdGenerator.incrementAndGet(), "replication-metadata-" + dataNodeId.getHostname() + "[" + dataNodeId.getDatacenterName() + "]", replicaMetadataRequestInfoList, replicationConfig.replicationFetchSizeInBytes, replicationConfig.replicaMetadataRequestVersion);
        channelOutput = connectedChannel.sendAndReceive(request);
        logger.trace("Remote node: {} Thread name: {} Remote replicas: {} Stream size after deserialization: {} ", remoteNode, threadName, replicasToReplicatePerNode, channelOutput.getInputStream().available());
        ReplicaMetadataResponse response = ReplicaMetadataResponse.readFrom(channelOutput.getInputStream(), findTokenHelper, clusterMap);
        long metadataRequestTime = time.milliseconds() - replicaMetadataRequestStartTime;
        replicationMetrics.updateMetadataRequestTime(metadataRequestTime, replicatingFromRemoteColo, replicatingOverSsl, datacenterName);
        if (response.getError() != ServerErrorCode.No_Error || response.getReplicaMetadataResponseInfoList().size() != replicasToReplicatePerNode.size()) {
            int replicaMetadataResponseInfoListSize = response.getReplicaMetadataResponseInfoList() == null ? 0 : response.getReplicaMetadataResponseInfoList().size();
            logger.error("Remote node: {} Thread name: {} Remote replicas: {} Replica metadata response error: {} ReplicaMetadataResponseInfoListSize: {} ReplicasToReplicatePerNodeSize: {}", remoteNode, threadName, replicasToReplicatePerNode, response.getError(), replicaMetadataResponseInfoListSize, replicasToReplicatePerNode.size());
            throw new ReplicationException("Replica Metadata Response Error " + response.getError());
        }
        return response;
    } catch (Exception e) {
        responseHandler.onEvent(replicasToReplicatePerNode.get(0).getReplicaId(), e);
        throw e;
    } finally {
        if (channelOutput != null && (channelOutput.getInputStream() instanceof NettyByteBufDataInputStream)) {
            // Release buffer if and only if the inputStream is NettyByteBuf based.
            ((NettyByteBufDataInputStream) channelOutput.getInputStream()).getBuffer().release();
        }
    }
}

Example 13 with NettyByteBufDataInputStream

use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.

the class RequestResponseTest method serAndPrepForRead.

/**
 * Serializes a {@link RequestOrResponseType} and prepares it for reading.
 * @param requestOrResponse the {@link RequestOrResponseType} to serialize.
 * @param channelSize the amount of data that the output channel should read in one iteration. Setting this to -1
 *                       will set the size of the output channel buffer to 1/3rd the size of {@code requestOrResponse}
 * @param isRequest {@code true} if {@code requestOrResponse} is a request. {@code false} otherwise.
 * @return the serialized form of {@code requestOrResponse} as a {@link DataInputStream}.
 * @throws IOException
 */
private DataInputStream serAndPrepForRead(RequestOrResponse requestOrResponse, int channelSize, boolean isRequest) throws IOException {
    DataInputStream stream;
    if (useByteBufContent && requestOrResponse.content() != null) {
        stream = new NettyByteBufDataInputStream(requestOrResponse.content());
    } else {
        if (channelSize == -1) {
            channelSize = (int) (requestOrResponse.sizeInBytes() / 3);
        }
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        int expectedWriteToCount = (int) ((requestOrResponse.sizeInBytes() + channelSize - 1) / channelSize);
        int actualWriteToCount = 0;
        do {
            ByteBufferChannel channel = new ByteBufferChannel(ByteBuffer.allocate(channelSize));
            requestOrResponse.writeTo(channel);
            ByteBuffer underlyingBuf = channel.getBuffer();
            underlyingBuf.flip();
            outputStream.write(underlyingBuf.array(), underlyingBuf.arrayOffset(), underlyingBuf.remaining());
            actualWriteToCount++;
        } while (!requestOrResponse.isSendComplete());
        Assert.assertEquals("Should not have written anything", 0, requestOrResponse.writeTo(new ByteBufferChannel(ByteBuffer.allocate(1))));
        Assert.assertEquals("writeTo() should have written out as much as the channel could take in every call", expectedWriteToCount, actualWriteToCount);
        stream = new DataInputStream(new ByteArrayInputStream(outputStream.toByteArray()));
    }
    // read length
    stream.readLong();
    if (isRequest) {
        // read version
        Assert.assertEquals(RequestOrResponseType.values()[stream.readShort()], requestOrResponse.getRequestType());
    }
    return stream;
}

Example 14 with NettyByteBufDataInputStream

use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.

the class GetBlobOperationTest method testTimeoutAndBlobNotFoundInOriginDc.

/**
 * Test the case where origin replicas return Blob_Not_found and the rest times out.
 * @throws Exception
 */
@Test
public void testTimeoutAndBlobNotFoundInOriginDc() throws Exception {
    assumeTrue(operationTrackerType.equals(AdaptiveOperationTracker.class.getSimpleName()));
    doPut();
    // Pick a remote DC as the new local DC.
    String newLocal = "DC1";
    String oldLocal = localDcName;
    Properties props = getDefaultNonBlockingRouterProperties(true);
    props.setProperty("router.datacenter.name", newLocal);
    props.setProperty("router.get.request.parallelism", "3");
    props.setProperty("router.operation.tracker.max.inflight.requests", "3");
    routerConfig = new RouterConfig(new VerifiableProperties(props));
    GetBlobOperation op = createOperation(routerConfig, null);
    correlationIdToGetOperation.clear();
    for (MockServer server : mockServerLayout.getMockServers()) {
        server.setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
    }
    op.poll(requestRegistrationCallback);
    time.sleep(routerConfig.routerRequestTimeoutMs + 1);
    // The request should have response from all remote replicas.
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestRegistrationCallback.getRequestsToSend());
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content()), mockClusterMap) : null;
            op.handleResponse(responseInfo, getResponse);
            responseInfo.release();
        }
    }
    RouterException routerException = (RouterException) op.getOperationException();
    // error code should be OperationTimedOut because it precedes BlobDoesNotExist
    Assert.assertEquals(RouterErrorCode.BlobDoesNotExist, routerException.getErrorCode());
    props = getDefaultNonBlockingRouterProperties(true);
    props.setProperty("router.datacenter.name", oldLocal);
    props.setProperty("router.get.request.parallelism", "2");
    props.setProperty("router.operation.tracker.max.inflight.requests", "2");
    routerConfig = new RouterConfig(new VerifiableProperties(props));
}

Example 15 with NettyByteBufDataInputStream

use of com.github.ambry.utils.NettyByteBufDataInputStream in project ambry by linkedin.

the class RouterUtils method extractResponseAndNotifyResponseHandler.

/**
 * Extract the {@link Response} from the given {@link ResponseInfo}
 * @param <R> the {@link Response} type.
 * @param responseHandler the {@link ResponseHandler} instance to use.
 * @param routerMetrics the {@link NonBlockingRouterMetrics} instance to use.
 * @param responseInfo the {@link ResponseInfo} from which the {@link Response} is to be extracted.
 * @param deserializer the {@link Deserializer} to use.
 * @param errorExtractor extract the {@link ServerErrorCode} to send to {@link ResponseHandler#onEvent}.
 * @return the extracted {@link Response} if there is one; null otherwise.
 */
@SuppressWarnings("unchecked")
static <R extends Response> R extractResponseAndNotifyResponseHandler(ResponseHandler responseHandler, NonBlockingRouterMetrics routerMetrics, ResponseInfo responseInfo, Deserializer<R> deserializer, Function<R, ServerErrorCode> errorExtractor) {
    R response = null;
    ReplicaId replicaId = responseInfo.getRequestInfo().getReplicaId();
    NetworkClientErrorCode networkClientErrorCode = responseInfo.getError();
    if (networkClientErrorCode == null) {
        try {
            if (responseInfo.getResponse() != null) {
                // If this responseInfo already has the deserialized java object, we can reference it directly. This is
                // applicable when we are receiving responses from Azure APIs in frontend. The responses from Azure are
                // handled in {@code AmbryRequest} class methods using a thread pool running with in the frontend. These
                // responses are then sent using local queues in {@code LocalRequestResponseChannel}.
                response = (R) mapToReceivedResponse((Response) responseInfo.getResponse());
            } else {
                DataInputStream dis = new NettyByteBufDataInputStream(responseInfo.content());
                response = deserializer.readFrom(dis);
            }
            responseHandler.onEvent(replicaId, errorExtractor.apply(response));
        } catch (Exception e) {
            // Ignore. There is no value in notifying the response handler.
            logger.error("Response deserialization received unexpected error", e);
            routerMetrics.responseDeserializationErrorCount.inc();
        }
    } else {
        responseHandler.onEvent(replicaId, networkClientErrorCode);
    }
    return response;
}