Examples with FetchSnapshotResponseData org.apache.kafka.common.message.FetchSnapshotResponseData used on opensource projects

Example 1 with FetchSnapshotResponseData

use of org.apache.kafka.common.message.FetchSnapshotResponseData in project kafka by apache.

the class RaftClientTestContext method assertSentFetchSnapshotResponse.

Optional<FetchSnapshotResponseData.PartitionSnapshot> assertSentFetchSnapshotResponse(TopicPartition topicPartition) {
    List<RaftResponse.Outbound> sentMessages = drainSentResponses(ApiKeys.FETCH_SNAPSHOT);
    assertEquals(1, sentMessages.size());
    RaftMessage message = sentMessages.get(0);
    assertTrue(message.data() instanceof FetchSnapshotResponseData);
    FetchSnapshotResponseData response = (FetchSnapshotResponseData) message.data();
    assertEquals(Errors.NONE, Errors.forCode(response.errorCode()));
    return FetchSnapshotResponse.forTopicPartition(response, topicPartition);
}

Example 2 with FetchSnapshotResponseData

use of org.apache.kafka.common.message.FetchSnapshotResponseData in project kafka by apache.

the class RaftClientTestContext method assertSentFetchSnapshotResponse.

void assertSentFetchSnapshotResponse(Errors responseError) {
    List<RaftResponse.Outbound> sentMessages = drainSentResponses(ApiKeys.FETCH_SNAPSHOT);
    assertEquals(1, sentMessages.size());
    RaftMessage message = sentMessages.get(0);
    assertTrue(message.data() instanceof FetchSnapshotResponseData);
    FetchSnapshotResponseData response = (FetchSnapshotResponseData) message.data();
    assertEquals(responseError, Errors.forCode(response.errorCode()));
}

Example 3 with FetchSnapshotResponseData

use of org.apache.kafka.common.message.FetchSnapshotResponseData in project kafka by apache.

the class KafkaRaftClient method handleFetchSnapshotRequest.

/**
 * Handle a FetchSnapshot request, similar to the Fetch request but we use {@link UnalignedRecords}
 * in response because the records are not necessarily offset-aligned.
 *
 * This API may return the following errors:
 *
 * - {@link Errors#INCONSISTENT_CLUSTER_ID} if the cluster id is presented in request
 *     but different from this node
 * - {@link Errors#BROKER_NOT_AVAILABLE} if this node is currently shutting down
 * - {@link Errors#FENCED_LEADER_EPOCH} if the epoch is smaller than this node's epoch
 * - {@link Errors#INVALID_REQUEST} if the request epoch is larger than the leader's current epoch
 *     or if either the fetch offset or the last fetched epoch is invalid
 * - {@link Errors#SNAPSHOT_NOT_FOUND} if the request snapshot id does not exists
 * - {@link Errors#POSITION_OUT_OF_RANGE} if the request snapshot offset out of range
 */
private FetchSnapshotResponseData handleFetchSnapshotRequest(RaftRequest.Inbound requestMetadata) {
    FetchSnapshotRequestData data = (FetchSnapshotRequestData) requestMetadata.data;
    if (!hasValidClusterId(data.clusterId())) {
        return new FetchSnapshotResponseData().setErrorCode(Errors.INCONSISTENT_CLUSTER_ID.code());
    }
    if (data.topics().size() != 1 && data.topics().get(0).partitions().size() != 1) {
        return FetchSnapshotResponse.withTopLevelError(Errors.INVALID_REQUEST);
    }
    Optional<FetchSnapshotRequestData.PartitionSnapshot> partitionSnapshotOpt = FetchSnapshotRequest.forTopicPartition(data, log.topicPartition());
    if (!partitionSnapshotOpt.isPresent()) {
        // The Raft client assumes that there is only one topic partition.
        TopicPartition unknownTopicPartition = new TopicPartition(data.topics().get(0).name(), data.topics().get(0).partitions().get(0).partition());
        return FetchSnapshotResponse.singleton(unknownTopicPartition, responsePartitionSnapshot -> responsePartitionSnapshot.setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()));
    }
    FetchSnapshotRequestData.PartitionSnapshot partitionSnapshot = partitionSnapshotOpt.get();
    Optional<Errors> leaderValidation = validateLeaderOnlyRequest(partitionSnapshot.currentLeaderEpoch());
    if (leaderValidation.isPresent()) {
        return FetchSnapshotResponse.singleton(log.topicPartition(), responsePartitionSnapshot -> addQuorumLeader(responsePartitionSnapshot).setErrorCode(leaderValidation.get().code()));
    }
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(partitionSnapshot.snapshotId().endOffset(), partitionSnapshot.snapshotId().epoch());
    Optional<RawSnapshotReader> snapshotOpt = log.readSnapshot(snapshotId);
    if (!snapshotOpt.isPresent()) {
        return FetchSnapshotResponse.singleton(log.topicPartition(), responsePartitionSnapshot -> addQuorumLeader(responsePartitionSnapshot).setErrorCode(Errors.SNAPSHOT_NOT_FOUND.code()));
    }
    RawSnapshotReader snapshot = snapshotOpt.get();
    long snapshotSize = snapshot.sizeInBytes();
    if (partitionSnapshot.position() < 0 || partitionSnapshot.position() >= snapshotSize) {
        return FetchSnapshotResponse.singleton(log.topicPartition(), responsePartitionSnapshot -> addQuorumLeader(responsePartitionSnapshot).setErrorCode(Errors.POSITION_OUT_OF_RANGE.code()));
    }
    if (partitionSnapshot.position() > Integer.MAX_VALUE) {
        throw new IllegalStateException(String.format("Trying to fetch a snapshot with size (%s) and a position (%s) larger than %s", snapshotSize, partitionSnapshot.position(), Integer.MAX_VALUE));
    }
    int maxSnapshotSize;
    try {
        maxSnapshotSize = Math.toIntExact(snapshotSize);
    } catch (ArithmeticException e) {
        maxSnapshotSize = Integer.MAX_VALUE;
    }
    UnalignedRecords records = snapshot.slice(partitionSnapshot.position(), Math.min(data.maxBytes(), maxSnapshotSize));
    return FetchSnapshotResponse.singleton(log.topicPartition(), responsePartitionSnapshot -> {
        addQuorumLeader(responsePartitionSnapshot).snapshotId().setEndOffset(snapshotId.offset).setEpoch(snapshotId.epoch);
        return responsePartitionSnapshot.setSize(snapshotSize).setPosition(partitionSnapshot.position()).setUnalignedRecords(records);
    });
}

Example 4 with FetchSnapshotResponseData

use of org.apache.kafka.common.message.FetchSnapshotResponseData in project kafka by apache.

the class KafkaRaftClient method handleFetchSnapshotResponse.

private boolean handleFetchSnapshotResponse(RaftResponse.Inbound responseMetadata, long currentTimeMs) {
    FetchSnapshotResponseData data = (FetchSnapshotResponseData) responseMetadata.data;
    Errors topLevelError = Errors.forCode(data.errorCode());
    if (topLevelError != Errors.NONE) {
        return handleTopLevelError(topLevelError, responseMetadata);
    }
    if (data.topics().size() != 1 && data.topics().get(0).partitions().size() != 1) {
        return false;
    }
    Optional<FetchSnapshotResponseData.PartitionSnapshot> partitionSnapshotOpt = FetchSnapshotResponse.forTopicPartition(data, log.topicPartition());
    if (!partitionSnapshotOpt.isPresent()) {
        return false;
    }
    FetchSnapshotResponseData.PartitionSnapshot partitionSnapshot = partitionSnapshotOpt.get();
    FetchSnapshotResponseData.LeaderIdAndEpoch currentLeaderIdAndEpoch = partitionSnapshot.currentLeader();
    OptionalInt responseLeaderId = optionalLeaderId(currentLeaderIdAndEpoch.leaderId());
    int responseEpoch = currentLeaderIdAndEpoch.leaderEpoch();
    Errors error = Errors.forCode(partitionSnapshot.errorCode());
    Optional<Boolean> handled = maybeHandleCommonResponse(error, responseLeaderId, responseEpoch, currentTimeMs);
    if (handled.isPresent()) {
        return handled.get();
    }
    FollowerState state = quorum.followerStateOrThrow();
    if (Errors.forCode(partitionSnapshot.errorCode()) == Errors.SNAPSHOT_NOT_FOUND || partitionSnapshot.snapshotId().endOffset() < 0 || partitionSnapshot.snapshotId().epoch() < 0) {
        /* The leader deleted the snapshot before the follower could download it. Start over by
             * reseting the fetching snapshot state and sending another fetch request.
             */
        logger.trace("Leader doesn't know about snapshot id {}, returned error {} and snapshot id {}", state.fetchingSnapshot(), partitionSnapshot.errorCode(), partitionSnapshot.snapshotId());
        state.setFetchingSnapshot(Optional.empty());
        state.resetFetchTimeout(currentTimeMs);
        return true;
    }
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(partitionSnapshot.snapshotId().endOffset(), partitionSnapshot.snapshotId().epoch());
    RawSnapshotWriter snapshot;
    if (state.fetchingSnapshot().isPresent()) {
        snapshot = state.fetchingSnapshot().get();
    } else {
        throw new IllegalStateException(String.format("Received unexpected fetch snapshot response: %s", partitionSnapshot));
    }
    if (!snapshot.snapshotId().equals(snapshotId)) {
        throw new IllegalStateException(String.format("Received fetch snapshot response with an invalid id. Expected %s; Received %s", snapshot.snapshotId(), snapshotId));
    }
    if (snapshot.sizeInBytes() != partitionSnapshot.position()) {
        throw new IllegalStateException(String.format("Received fetch snapshot response with an invalid position. Expected %s; Received %s", snapshot.sizeInBytes(), partitionSnapshot.position()));
    }
    final UnalignedMemoryRecords records;
    if (partitionSnapshot.unalignedRecords() instanceof MemoryRecords) {
        records = new UnalignedMemoryRecords(((MemoryRecords) partitionSnapshot.unalignedRecords()).buffer());
    } else if (partitionSnapshot.unalignedRecords() instanceof UnalignedMemoryRecords) {
        records = (UnalignedMemoryRecords) partitionSnapshot.unalignedRecords();
    } else {
        throw new IllegalStateException(String.format("Received unexpected fetch snapshot response: %s", partitionSnapshot));
    }
    snapshot.append(records);
    if (snapshot.sizeInBytes() == partitionSnapshot.size()) {
        // Finished fetching the snapshot.
        snapshot.freeze();
        state.setFetchingSnapshot(Optional.empty());
        if (log.truncateToLatestSnapshot()) {
            updateFollowerHighWatermark(state, OptionalLong.of(log.highWatermark().offset));
        } else {
            throw new IllegalStateException(String.format("Full log truncation expected but didn't happen. Snapshot of %s, log end offset %s, last fetched %s", snapshot.snapshotId(), log.endOffset(), log.lastFetchedEpoch()));
        }
    }
    state.resetFetchTimeout(currentTimeMs);
    return true;
}