Examples with FetchResponseData org.apache.kafka.common.message.FetchResponseData used on opensource projects

Example 6 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class RaftClientTestContext method assertSentFetchPartitionResponse.

FetchResponseData.PartitionData assertSentFetchPartitionResponse() {
    List<RaftResponse.Outbound> sentMessages = drainSentResponses(ApiKeys.FETCH);
    assertEquals(1, sentMessages.size(), "Found unexpected sent messages " + sentMessages);
    RaftResponse.Outbound raftMessage = sentMessages.get(0);
    assertEquals(ApiKeys.FETCH.id, raftMessage.data.apiKey());
    FetchResponseData response = (FetchResponseData) raftMessage.data();
    assertEquals(Errors.NONE, Errors.forCode(response.errorCode()));
    assertEquals(1, response.responses().size());
    assertEquals(metadataPartition.topic(), response.responses().get(0).topic());
    assertEquals(1, response.responses().get(0).partitions().size());
    return response.responses().get(0).partitions().get(0);
}

Example 7 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClient method handleFetchResponse.

private boolean handleFetchResponse(RaftResponse.Inbound responseMetadata, long currentTimeMs) {
    FetchResponseData response = (FetchResponseData) responseMetadata.data;
    Errors topLevelError = Errors.forCode(response.errorCode());
    if (topLevelError != Errors.NONE) {
        return handleTopLevelError(topLevelError, responseMetadata);
    }
    if (!RaftUtil.hasValidTopicPartition(response, log.topicPartition(), log.topicId())) {
        return false;
    }
    // If the ID is valid, we can set the topic name.
    response.responses().get(0).setTopic(log.topicPartition().topic());
    FetchResponseData.PartitionData partitionResponse = response.responses().get(0).partitions().get(0);
    FetchResponseData.LeaderIdAndEpoch currentLeaderIdAndEpoch = partitionResponse.currentLeader();
    OptionalInt responseLeaderId = optionalLeaderId(currentLeaderIdAndEpoch.leaderId());
    int responseEpoch = currentLeaderIdAndEpoch.leaderEpoch();
    Errors error = Errors.forCode(partitionResponse.errorCode());
    Optional<Boolean> handled = maybeHandleCommonResponse(error, responseLeaderId, responseEpoch, currentTimeMs);
    if (handled.isPresent()) {
        return handled.get();
    }
    FollowerState state = quorum.followerStateOrThrow();
    if (error == Errors.NONE) {
        FetchResponseData.EpochEndOffset divergingEpoch = partitionResponse.divergingEpoch();
        if (divergingEpoch.epoch() >= 0) {
            // The leader is asking us to truncate before continuing
            final OffsetAndEpoch divergingOffsetAndEpoch = new OffsetAndEpoch(divergingEpoch.endOffset(), divergingEpoch.epoch());
            state.highWatermark().ifPresent(highWatermark -> {
                if (divergingOffsetAndEpoch.offset < highWatermark.offset) {
                    throw new KafkaException("The leader requested truncation to offset " + divergingOffsetAndEpoch.offset + ", which is below the current high watermark" + " " + highWatermark);
                }
            });
            long truncationOffset = log.truncateToEndOffset(divergingOffsetAndEpoch);
            logger.info("Truncated to offset {} from Fetch response from leader {}", truncationOffset, quorum.leaderIdOrSentinel());
        } else if (partitionResponse.snapshotId().epoch() >= 0 || partitionResponse.snapshotId().endOffset() >= 0) {
            if (partitionResponse.snapshotId().epoch() < 0) {
                logger.error("The leader sent a snapshot id with a valid end offset {} but with an invalid epoch {}", partitionResponse.snapshotId().endOffset(), partitionResponse.snapshotId().epoch());
                return false;
            } else if (partitionResponse.snapshotId().endOffset() < 0) {
                logger.error("The leader sent a snapshot id with a valid epoch {} but with an invalid end offset {}", partitionResponse.snapshotId().epoch(), partitionResponse.snapshotId().endOffset());
                return false;
            } else {
                final OffsetAndEpoch snapshotId = new OffsetAndEpoch(partitionResponse.snapshotId().endOffset(), partitionResponse.snapshotId().epoch());
                // Do not validate the snapshot id against the local replicated log
                // since this snapshot is expected to reference offsets and epochs
                // greater than the log end offset and high-watermark
                state.setFetchingSnapshot(log.storeSnapshot(snapshotId));
            }
        } else {
            Records records = FetchResponse.recordsOrFail(partitionResponse);
            if (records.sizeInBytes() > 0) {
                appendAsFollower(records);
            }
            OptionalLong highWatermark = partitionResponse.highWatermark() < 0 ? OptionalLong.empty() : OptionalLong.of(partitionResponse.highWatermark());
            updateFollowerHighWatermark(state, highWatermark);
        }
        state.resetFetchTimeout(currentTimeMs);
        return true;
    } else {
        return handleUnexpectedError(error, responseMetadata);
    }
}

Example 8 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClientTest method testFollowerReplication.

@Test
public void testFollowerReplication() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, otherNodeId).build();
    context.assertElectedLeader(epoch, otherNodeId);
    context.pollUntilRequest();
    int fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    Records records = context.buildBatch(0L, 3, Arrays.asList("a", "b"));
    FetchResponseData response = context.fetchResponse(epoch, otherNodeId, records, 0L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, response);
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    assertEquals(2L, context.log.lastFlushedOffset());
}

Example 9 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClientTest method testClusterAuthorizationFailedInFetch.

@Test
public void testClusterAuthorizationFailedInFetch() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, otherNodeId).build();
    context.assertElectedLeader(epoch, otherNodeId);
    context.pollUntilRequest();
    int correlationId = context.assertSentFetchRequest(epoch, 0, 0);
    FetchResponseData response = new FetchResponseData().setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code());
    context.deliverResponse(correlationId, otherNodeId, response);
    assertThrows(ClusterAuthorizationException.class, context.client::poll);
}

Example 10 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClientTest method testEmptyRecordSetInFetchResponse.

@Test
public void testEmptyRecordSetInFetchResponse() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, otherNodeId).build();
    context.assertElectedLeader(epoch, otherNodeId);
    // Receive an empty fetch response
    context.pollUntilRequest();
    int fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    FetchResponseData fetchResponse = context.fetchResponse(epoch, otherNodeId, MemoryRecords.EMPTY, 0L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, fetchResponse);
    context.client.poll();
    assertEquals(0L, context.log.endOffset().offset);
    assertEquals(OptionalLong.of(0L), context.client.highWatermark());
    // Receive some records in the next poll, but do not advance high watermark
    context.pollUntilRequest();
    Records records = context.buildBatch(0L, epoch, Arrays.asList("a", "b"));
    fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    fetchResponse = context.fetchResponse(epoch, otherNodeId, records, 0L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, fetchResponse);
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    assertEquals(OptionalLong.of(0L), context.client.highWatermark());
    // The next fetch response is empty, but should still advance the high watermark
    context.pollUntilRequest();
    fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 2L, epoch);
    fetchResponse = context.fetchResponse(epoch, otherNodeId, MemoryRecords.EMPTY, 2L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, fetchResponse);
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    assertEquals(OptionalLong.of(2L), context.client.highWatermark());
}