Examples with PartitionId com.github.ambry.clustermap.PartitionId used on opensource projects

Example 1 with PartitionId

use of com.github.ambry.clustermap.PartitionId in project ambry by linkedin.

the class ReplicationManager method readFromFileAndPersistIfNecessary.

/**
 * Reads the replica tokens from the file and populates the Remote replica info
 * and persists the token file if necessary.
 * @param mountPath The mount path where the replica tokens are stored
 * @throws ReplicationException
 * @throws IOException
 */
private void readFromFileAndPersistIfNecessary(String mountPath) throws ReplicationException, IOException {
    logger.info("Reading replica tokens for mount path {}", mountPath);
    long readStartTimeMs = SystemTime.getInstance().milliseconds();
    File replicaTokenFile = new File(mountPath, replicaTokenFileName);
    boolean tokenWasReset = false;
    if (replicaTokenFile.exists()) {
        CrcInputStream crcStream = new CrcInputStream(new FileInputStream(replicaTokenFile));
        DataInputStream stream = new DataInputStream(crcStream);
        try {
            short version = stream.readShort();
            switch(version) {
                case 0:
                    while (stream.available() > Crc_Size) {
                        // read partition id
                        PartitionId partitionId = clusterMap.getPartitionIdFromStream(stream);
                        // read remote node host name
                        String hostname = Utils.readIntString(stream);
                        // read remote replica path
                        String replicaPath = Utils.readIntString(stream);
                        // read remote port
                        int port = stream.readInt();
                        // read total bytes read from local store
                        long totalBytesReadFromLocalStore = stream.readLong();
                        // read replica token
                        FindToken token = factory.getFindToken(stream);
                        // update token
                        PartitionInfo partitionInfo = partitionsToReplicate.get(partitionId);
                        if (partitionInfo != null) {
                            boolean updatedToken = false;
                            for (RemoteReplicaInfo remoteReplicaInfo : partitionInfo.getRemoteReplicaInfos()) {
                                if (remoteReplicaInfo.getReplicaId().getDataNodeId().getHostname().equalsIgnoreCase(hostname) && remoteReplicaInfo.getReplicaId().getDataNodeId().getPort() == port && remoteReplicaInfo.getReplicaId().getReplicaPath().equals(replicaPath)) {
                                    logger.info("Read token for partition {} remote host {} port {} token {}", partitionId, hostname, port, token);
                                    if (partitionInfo.getStore().getSizeInBytes() > 0) {
                                        remoteReplicaInfo.initializeTokens(token);
                                        remoteReplicaInfo.setTotalBytesReadFromLocalStore(totalBytesReadFromLocalStore);
                                    } else {
                                        // if the local replica is empty, it could have been newly created. In this case, the offset in
                                        // every peer replica which the local replica lags from should be set to 0, so that the local
                                        // replica starts fetching from the beginning of the peer. The totalBytes the peer read from the
                                        // local replica should also be set to 0. During initialization these values are already set to 0,
                                        // so we let them be.
                                        tokenWasReset = true;
                                        logTokenReset(partitionId, hostname, port, token);
                                    }
                                    updatedToken = true;
                                    break;
                                }
                            }
                            if (!updatedToken) {
                                logger.warn("Persisted remote replica host {} and port {} not present in new cluster ", hostname, port);
                            }
                        } else {
                            // If this partition was not found in partitionsToReplicate, it means that the local store corresponding
                            // to this partition could not be started. In such a case, the tokens for its remote replicas should be
                            // reset.
                            tokenWasReset = true;
                            logTokenReset(partitionId, hostname, port, token);
                        }
                    }
                    long crc = crcStream.getValue();
                    if (crc != stream.readLong()) {
                        throw new ReplicationException("Crc check does not match for replica token file for mount path " + mountPath);
                    }
                    break;
                default:
                    throw new ReplicationException("Invalid version in replica token file for mount path " + mountPath);
            }
        } catch (IOException e) {
            throw new ReplicationException("IO error while reading from replica token file " + e);
        } finally {
            stream.close();
            replicationMetrics.remoteReplicaTokensRestoreTime.update(SystemTime.getInstance().milliseconds() - readStartTimeMs);
        }
    }
    if (tokenWasReset) {
        // We must ensure that the the token file is persisted if any of the tokens in the file got reset. We need to do
        // this before an associated store takes any writes, to avoid the case where a store takes writes and persists it,
        // before the replica token file is persisted after the reset.
        persistor.write(mountPath, false);
    }
}

Example 2 with PartitionId

use of com.github.ambry.clustermap.PartitionId in project ambry by linkedin.

the class AmbryRequests method handleCatchupStatusRequest.

/**
 * Handles {@link com.github.ambry.protocol.AdminRequestOrResponseType#CatchupStatus}.
 * @param requestStream the serialized bytes of the request.
 * @param adminRequest the {@link AdminRequest} received.
 * @return the {@link AdminResponse} to the request.
 * @throws IOException if there is any I/O error reading from the {@code requestStream}.
 */
private AdminResponse handleCatchupStatusRequest(DataInputStream requestStream, AdminRequest adminRequest) throws IOException {
    Collection<PartitionId> partitionIds;
    ServerErrorCode error = ServerErrorCode.No_Error;
    boolean isCaughtUp = false;
    CatchupStatusAdminRequest catchupStatusRequest = CatchupStatusAdminRequest.readFrom(requestStream, adminRequest);
    if (catchupStatusRequest.getAcceptableLagInBytes() < 0) {
        error = ServerErrorCode.Bad_Request;
    } else if (catchupStatusRequest.getNumReplicasCaughtUpPerPartition() <= 0) {
        error = ServerErrorCode.Bad_Request;
    } else {
        if (catchupStatusRequest.getPartitionId() != null) {
            error = validateRequest(catchupStatusRequest.getPartitionId(), RequestOrResponseType.AdminRequest);
            partitionIds = Collections.singletonList(catchupStatusRequest.getPartitionId());
        } else {
            partitionIds = partitionsInCurrentNode;
        }
        if (!error.equals(ServerErrorCode.Partition_Unknown)) {
            error = ServerErrorCode.No_Error;
            isCaughtUp = isRemoteLagLesserOrEqual(partitionIds, catchupStatusRequest.getAcceptableLagInBytes(), catchupStatusRequest.getNumReplicasCaughtUpPerPartition());
        }
    }
    AdminResponse adminResponse = new AdminResponse(adminRequest.getCorrelationId(), adminRequest.getClientId(), error);
    return new CatchupStatusAdminResponse(isCaughtUp, adminResponse);
}

Example 3 with PartitionId

use of com.github.ambry.clustermap.PartitionId in project ambry by linkedin.

the class AmbryRequests method handleRequestControlRequest.

/**
 * Handles {@link com.github.ambry.protocol.AdminRequestOrResponseType#RequestControl}.
 * @param requestStream the serialized bytes of the request.
 * @param adminRequest the {@link AdminRequest} received.
 * @return the {@link AdminResponse} to the request.
 * @throws IOException if there is any I/O error reading from the {@code requestStream}.
 */
private AdminResponse handleRequestControlRequest(DataInputStream requestStream, AdminRequest adminRequest) throws IOException {
    RequestControlAdminRequest controlRequest = RequestControlAdminRequest.readFrom(requestStream, adminRequest);
    RequestOrResponseType toControl = controlRequest.getRequestTypeToControl();
    ServerErrorCode error;
    Collection<PartitionId> partitionIds;
    if (!requestsDisableInfo.containsKey(toControl)) {
        metrics.badRequestError.inc();
        error = ServerErrorCode.Bad_Request;
    } else {
        error = ServerErrorCode.No_Error;
        if (controlRequest.getPartitionId() != null) {
            error = validateRequest(controlRequest.getPartitionId(), RequestOrResponseType.AdminRequest);
            partitionIds = Collections.singletonList(controlRequest.getPartitionId());
        } else {
            partitionIds = partitionsInCurrentNode;
        }
        if (!error.equals(ServerErrorCode.Partition_Unknown)) {
            controlRequestForPartitions(toControl, partitionIds, controlRequest.shouldEnable());
            for (PartitionId partitionId : partitionIds) {
                logger.info("Enable state for {} on {} is {}", toControl, partitionId, isRequestEnabled(toControl, partitionId));
            }
        }
    }
    return new AdminResponse(adminRequest.getCorrelationId(), adminRequest.getClientId(), error);
}

Example 4 with PartitionId

use of com.github.ambry.clustermap.PartitionId in project ambry by linkedin.

the class AmbryRequests method handleReplicaMetadataRequest.

public void handleReplicaMetadataRequest(Request request) throws IOException, InterruptedException {
    ReplicaMetadataRequest replicaMetadataRequest = ReplicaMetadataRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap, findTokenFactory);
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.replicaMetadataRequestQueueTimeInMs.update(requestQueueTime);
    metrics.replicaMetadataRequestRate.mark();
    List<ReplicaMetadataRequestInfo> replicaMetadataRequestInfoList = replicaMetadataRequest.getReplicaMetadataRequestInfoList();
    int partitionCnt = replicaMetadataRequestInfoList.size();
    long startTimeInMs = SystemTime.getInstance().milliseconds();
    ReplicaMetadataResponse response = null;
    try {
        List<ReplicaMetadataResponseInfo> replicaMetadataResponseList = new ArrayList<ReplicaMetadataResponseInfo>(partitionCnt);
        for (ReplicaMetadataRequestInfo replicaMetadataRequestInfo : replicaMetadataRequestInfoList) {
            long partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
            PartitionId partitionId = replicaMetadataRequestInfo.getPartitionId();
            ServerErrorCode error = validateRequest(partitionId, RequestOrResponseType.ReplicaMetadataRequest);
            logger.trace("{} Time used to validate metadata request: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
            if (error != ServerErrorCode.No_Error) {
                logger.error("Validating replica metadata request failed with error {} for partition {}", error, partitionId);
                ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, error);
                replicaMetadataResponseList.add(replicaMetadataResponseInfo);
            } else {
                try {
                    FindToken findToken = replicaMetadataRequestInfo.getToken();
                    String hostName = replicaMetadataRequestInfo.getHostName();
                    String replicaPath = replicaMetadataRequestInfo.getReplicaPath();
                    Store store = storageManager.getStore(partitionId);
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    FindInfo findInfo = store.findEntriesSince(findToken, replicaMetadataRequest.getMaxTotalSizeOfEntriesInBytes());
                    logger.trace("{} Time used to find entry since: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    replicationManager.updateTotalBytesReadByRemoteReplica(partitionId, hostName, replicaPath, findInfo.getFindToken().getBytesRead());
                    logger.trace("{} Time used to update total bytes read: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    long remoteReplicaLagInBytes = replicationManager.getRemoteReplicaLagFromLocalInBytes(partitionId, hostName, replicaPath);
                    logger.trace("{} Time used to get remote replica lag in bytes: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, findInfo.getFindToken(), findInfo.getMessageEntries(), remoteReplicaLagInBytes);
                    replicaMetadataResponseList.add(replicaMetadataResponseInfo);
                } catch (StoreException e) {
                    logger.error("Store exception on a replica metadata request with error code " + e.getErrorCode() + " for partition " + partitionId, e);
                    if (e.getErrorCode() == StoreErrorCodes.IOError) {
                        metrics.storeIOError.inc();
                    } else {
                        metrics.unExpectedStoreFindEntriesError.inc();
                    }
                    ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
                    replicaMetadataResponseList.add(replicaMetadataResponseInfo);
                }
            }
        }
        response = new ReplicaMetadataResponse(replicaMetadataRequest.getCorrelationId(), replicaMetadataRequest.getClientId(), ServerErrorCode.No_Error, replicaMetadataResponseList);
    } catch (Exception e) {
        logger.error("Unknown exception for request " + replicaMetadataRequest, e);
        response = new ReplicaMetadataResponse(replicaMetadataRequest.getCorrelationId(), replicaMetadataRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTimeInMs;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", replicaMetadataRequest, response, processingTime);
        logger.trace("{} {} processingTime {}", replicaMetadataRequest, response, processingTime);
        metrics.replicaMetadataRequestProcessingTimeInMs.update(processingTime);
    }
    requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.replicaMetadataResponseQueueTimeInMs, metrics.replicaMetadataSendTimeInMs, metrics.replicaMetadataTotalTimeInMs, null, null, totalTimeSpent));
}

Example 5 with PartitionId

use of com.github.ambry.clustermap.PartitionId in project ambry by linkedin.

the class AmbryRequests method handleReplicationControlRequest.

/**
 * Handles {@link com.github.ambry.protocol.AdminRequestOrResponseType#ReplicationControl}.
 * @param requestStream the serialized bytes of the request.
 * @param adminRequest the {@link AdminRequest} received.
 * @return the {@link AdminResponse} to the request.
 * @throws IOException if there is any I/O error reading from the {@code requestStream}.
 */
private AdminResponse handleReplicationControlRequest(DataInputStream requestStream, AdminRequest adminRequest) throws IOException {
    Collection<PartitionId> partitionIds;
    ServerErrorCode error = ServerErrorCode.No_Error;
    ReplicationControlAdminRequest replControlRequest = ReplicationControlAdminRequest.readFrom(requestStream, adminRequest);
    if (replControlRequest.getPartitionId() != null) {
        error = validateRequest(replControlRequest.getPartitionId(), RequestOrResponseType.AdminRequest);
        partitionIds = Collections.singletonList(replControlRequest.getPartitionId());
    } else {
        partitionIds = partitionsInCurrentNode;
    }
    if (!error.equals(ServerErrorCode.Partition_Unknown)) {
        if (replicationManager.controlReplicationForPartitions(partitionIds, replControlRequest.getOrigins(), replControlRequest.shouldEnable())) {
            error = ServerErrorCode.No_Error;
        } else {
            logger.error("Could not set enable status for replication of {} from {} to {}. Check partition validity and" + " origins list", partitionIds, replControlRequest.getOrigins(), replControlRequest.shouldEnable());
            error = ServerErrorCode.Bad_Request;
        }
    }
    return new AdminResponse(adminRequest.getCorrelationId(), adminRequest.getClientId(), error);
}