Examples with FindToken com.github.ambry.store.FindToken used on opensource projects

Example 1 with FindToken

use of com.github.ambry.store.FindToken in project ambry by linkedin.

the class ReplicationManager method readFromFileAndPersistIfNecessary.

/**
 * Reads the replica tokens from the file and populates the Remote replica info
 * and persists the token file if necessary.
 * @param mountPath The mount path where the replica tokens are stored
 * @throws ReplicationException
 * @throws IOException
 */
private void readFromFileAndPersistIfNecessary(String mountPath) throws ReplicationException, IOException {
    logger.info("Reading replica tokens for mount path {}", mountPath);
    long readStartTimeMs = SystemTime.getInstance().milliseconds();
    File replicaTokenFile = new File(mountPath, replicaTokenFileName);
    boolean tokenWasReset = false;
    if (replicaTokenFile.exists()) {
        CrcInputStream crcStream = new CrcInputStream(new FileInputStream(replicaTokenFile));
        DataInputStream stream = new DataInputStream(crcStream);
        try {
            short version = stream.readShort();
            switch(version) {
                case 0:
                    while (stream.available() > Crc_Size) {
                        // read partition id
                        PartitionId partitionId = clusterMap.getPartitionIdFromStream(stream);
                        // read remote node host name
                        String hostname = Utils.readIntString(stream);
                        // read remote replica path
                        String replicaPath = Utils.readIntString(stream);
                        // read remote port
                        int port = stream.readInt();
                        // read total bytes read from local store
                        long totalBytesReadFromLocalStore = stream.readLong();
                        // read replica token
                        FindToken token = factory.getFindToken(stream);
                        // update token
                        PartitionInfo partitionInfo = partitionsToReplicate.get(partitionId);
                        if (partitionInfo != null) {
                            boolean updatedToken = false;
                            for (RemoteReplicaInfo remoteReplicaInfo : partitionInfo.getRemoteReplicaInfos()) {
                                if (remoteReplicaInfo.getReplicaId().getDataNodeId().getHostname().equalsIgnoreCase(hostname) && remoteReplicaInfo.getReplicaId().getDataNodeId().getPort() == port && remoteReplicaInfo.getReplicaId().getReplicaPath().equals(replicaPath)) {
                                    logger.info("Read token for partition {} remote host {} port {} token {}", partitionId, hostname, port, token);
                                    if (partitionInfo.getStore().getSizeInBytes() > 0) {
                                        remoteReplicaInfo.initializeTokens(token);
                                        remoteReplicaInfo.setTotalBytesReadFromLocalStore(totalBytesReadFromLocalStore);
                                    } else {
                                        // if the local replica is empty, it could have been newly created. In this case, the offset in
                                        // every peer replica which the local replica lags from should be set to 0, so that the local
                                        // replica starts fetching from the beginning of the peer. The totalBytes the peer read from the
                                        // local replica should also be set to 0. During initialization these values are already set to 0,
                                        // so we let them be.
                                        tokenWasReset = true;
                                        logTokenReset(partitionId, hostname, port, token);
                                    }
                                    updatedToken = true;
                                    break;
                                }
                            }
                            if (!updatedToken) {
                                logger.warn("Persisted remote replica host {} and port {} not present in new cluster ", hostname, port);
                            }
                        } else {
                            // If this partition was not found in partitionsToReplicate, it means that the local store corresponding
                            // to this partition could not be started. In such a case, the tokens for its remote replicas should be
                            // reset.
                            tokenWasReset = true;
                            logTokenReset(partitionId, hostname, port, token);
                        }
                    }
                    long crc = crcStream.getValue();
                    if (crc != stream.readLong()) {
                        throw new ReplicationException("Crc check does not match for replica token file for mount path " + mountPath);
                    }
                    break;
                default:
                    throw new ReplicationException("Invalid version in replica token file for mount path " + mountPath);
            }
        } catch (IOException e) {
            throw new ReplicationException("IO error while reading from replica token file " + e);
        } finally {
            stream.close();
            replicationMetrics.remoteReplicaTokensRestoreTime.update(SystemTime.getInstance().milliseconds() - readStartTimeMs);
        }
    }
    if (tokenWasReset) {
        // We must ensure that the the token file is persisted if any of the tokens in the file got reset. We need to do
        // this before an associated store takes any writes, to avoid the case where a store takes writes and persists it,
        // before the replica token file is persisted after the reset.
        persistor.write(mountPath, false);
    }
}

Example 2 with FindToken

use of com.github.ambry.store.FindToken in project ambry by linkedin.

the class AmbryRequests method handleReplicaMetadataRequest.

public void handleReplicaMetadataRequest(Request request) throws IOException, InterruptedException {
    ReplicaMetadataRequest replicaMetadataRequest = ReplicaMetadataRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap, findTokenFactory);
    long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
    long totalTimeSpent = requestQueueTime;
    metrics.replicaMetadataRequestQueueTimeInMs.update(requestQueueTime);
    metrics.replicaMetadataRequestRate.mark();
    List<ReplicaMetadataRequestInfo> replicaMetadataRequestInfoList = replicaMetadataRequest.getReplicaMetadataRequestInfoList();
    int partitionCnt = replicaMetadataRequestInfoList.size();
    long startTimeInMs = SystemTime.getInstance().milliseconds();
    ReplicaMetadataResponse response = null;
    try {
        List<ReplicaMetadataResponseInfo> replicaMetadataResponseList = new ArrayList<ReplicaMetadataResponseInfo>(partitionCnt);
        for (ReplicaMetadataRequestInfo replicaMetadataRequestInfo : replicaMetadataRequestInfoList) {
            long partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
            PartitionId partitionId = replicaMetadataRequestInfo.getPartitionId();
            ServerErrorCode error = validateRequest(partitionId, RequestOrResponseType.ReplicaMetadataRequest);
            logger.trace("{} Time used to validate metadata request: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
            if (error != ServerErrorCode.No_Error) {
                logger.error("Validating replica metadata request failed with error {} for partition {}", error, partitionId);
                ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, error);
                replicaMetadataResponseList.add(replicaMetadataResponseInfo);
            } else {
                try {
                    FindToken findToken = replicaMetadataRequestInfo.getToken();
                    String hostName = replicaMetadataRequestInfo.getHostName();
                    String replicaPath = replicaMetadataRequestInfo.getReplicaPath();
                    Store store = storageManager.getStore(partitionId);
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    FindInfo findInfo = store.findEntriesSince(findToken, replicaMetadataRequest.getMaxTotalSizeOfEntriesInBytes());
                    logger.trace("{} Time used to find entry since: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    replicationManager.updateTotalBytesReadByRemoteReplica(partitionId, hostName, replicaPath, findInfo.getFindToken().getBytesRead());
                    logger.trace("{} Time used to update total bytes read: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    partitionStartTimeInMs = SystemTime.getInstance().milliseconds();
                    long remoteReplicaLagInBytes = replicationManager.getRemoteReplicaLagFromLocalInBytes(partitionId, hostName, replicaPath);
                    logger.trace("{} Time used to get remote replica lag in bytes: {}", partitionId, (SystemTime.getInstance().milliseconds() - partitionStartTimeInMs));
                    ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, findInfo.getFindToken(), findInfo.getMessageEntries(), remoteReplicaLagInBytes);
                    replicaMetadataResponseList.add(replicaMetadataResponseInfo);
                } catch (StoreException e) {
                    logger.error("Store exception on a replica metadata request with error code " + e.getErrorCode() + " for partition " + partitionId, e);
                    if (e.getErrorCode() == StoreErrorCodes.IOError) {
                        metrics.storeIOError.inc();
                    } else {
                        metrics.unExpectedStoreFindEntriesError.inc();
                    }
                    ReplicaMetadataResponseInfo replicaMetadataResponseInfo = new ReplicaMetadataResponseInfo(partitionId, ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
                    replicaMetadataResponseList.add(replicaMetadataResponseInfo);
                }
            }
        }
        response = new ReplicaMetadataResponse(replicaMetadataRequest.getCorrelationId(), replicaMetadataRequest.getClientId(), ServerErrorCode.No_Error, replicaMetadataResponseList);
    } catch (Exception e) {
        logger.error("Unknown exception for request " + replicaMetadataRequest, e);
        response = new ReplicaMetadataResponse(replicaMetadataRequest.getCorrelationId(), replicaMetadataRequest.getClientId(), ServerErrorCode.Unknown_Error);
    } finally {
        long processingTime = SystemTime.getInstance().milliseconds() - startTimeInMs;
        totalTimeSpent += processingTime;
        publicAccessLogger.info("{} {} processingTime {}", replicaMetadataRequest, response, processingTime);
        logger.trace("{} {} processingTime {}", replicaMetadataRequest, response, processingTime);
        metrics.replicaMetadataRequestProcessingTimeInMs.update(processingTime);
    }
    requestResponseChannel.sendResponse(response, request, new ServerNetworkResponseMetrics(metrics.replicaMetadataResponseQueueTimeInMs, metrics.replicaMetadataSendTimeInMs, metrics.replicaMetadataTotalTimeInMs, null, null, totalTimeSpent));
}

Example 3 with FindToken

use of com.github.ambry.store.FindToken in project ambry by linkedin.

the class ReplicaMetadataRequestInfo method readFrom.

public static ReplicaMetadataRequestInfo readFrom(DataInputStream stream, ClusterMap clusterMap, FindTokenFactory factory) throws IOException {
    String hostName = Utils.readIntString(stream);
    String replicaPath = Utils.readIntString(stream);
    PartitionId partitionId = clusterMap.getPartitionIdFromStream(stream);
    FindToken token = factory.getFindToken(stream);
    return new ReplicaMetadataRequestInfo(partitionId, token, hostName, replicaPath);
}

Example 4 with FindToken

use of com.github.ambry.store.FindToken in project ambry by linkedin.

the class ReplicaMetadataResponseInfo method readFrom.

public static ReplicaMetadataResponseInfo readFrom(DataInputStream stream, FindTokenFactory factory, ClusterMap clusterMap, short replicaMetadataResponseVersion) throws IOException {
    PartitionId partitionId = clusterMap.getPartitionIdFromStream(stream);
    ServerErrorCode error = ServerErrorCode.values()[stream.readShort()];
    if (error != ServerErrorCode.No_Error) {
        return new ReplicaMetadataResponseInfo(partitionId, error);
    } else {
        FindToken token = factory.getFindToken(stream);
        Pair<List<MessageInfo>, List<MessageMetadata>> messageInfoAndMetadataList = MessageInfoAndMetadataListSerde.deserializeMessageInfoAndMetadataList(stream, clusterMap, getMessageInfoAndMetadataListSerDeVersion(replicaMetadataResponseVersion));
        long remoteReplicaLag = stream.readLong();
        return new ReplicaMetadataResponseInfo(partitionId, token, messageInfoAndMetadataList.getFirst(), remoteReplicaLag, replicaMetadataResponseVersion);
    }
}