Examples with FetchResponseData org.apache.kafka.common.message.FetchResponseData used on opensource projects

Example 11 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClientTest method testPurgatoryFetchCompletedByFollowerTransition.

@Test
public void testPurgatoryFetchCompletedByFollowerTransition() throws Exception {
    int localId = 0;
    int voter1 = localId;
    int voter2 = localId + 1;
    int voter3 = localId + 2;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(voter1, voter2, voter3);
    RaftClientTestContext context = RaftClientTestContext.initializeAsLeader(localId, voters, epoch);
    // Follower sends a fetch which cannot be satisfied immediately
    context.deliverRequest(context.fetchRequest(epoch, voter2, 1L, epoch, 500));
    context.client.poll();
    assertTrue(context.channel.drainSendQueue().stream().noneMatch(msg -> msg.data() instanceof FetchResponseData));
    // Now we get a BeginEpoch from the other voter and become a follower
    context.deliverRequest(context.beginEpochRequest(epoch + 1, voter3));
    context.pollUntilResponse();
    context.assertElectedLeader(epoch + 1, voter3);
    // We expect the BeginQuorumEpoch response and a failed Fetch response
    context.assertSentBeginQuorumEpochResponse(Errors.NONE, epoch + 1, OptionalInt.of(voter3));
    // The fetch should be satisfied immediately and return an error
    MemoryRecords fetchedRecords = context.assertSentFetchPartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER, epoch + 1, OptionalInt.of(voter3));
    assertEquals(0, fetchedRecords.sizeInBytes());
}

Example 12 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClientTest method testFetchResponseIgnoredAfterBecomingFollowerOfDifferentLeader.

@Test
public void testFetchResponseIgnoredAfterBecomingFollowerOfDifferentLeader() throws Exception {
    int localId = 0;
    int voter1 = localId;
    int voter2 = localId + 1;
    int voter3 = localId + 2;
    int epoch = 5;
    // Start out with `voter2` as the leader
    Set<Integer> voters = Utils.mkSet(voter1, voter2, voter3);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, voter2).build();
    context.assertElectedLeader(epoch, voter2);
    // Wait until we have a Fetch inflight to the leader
    context.pollUntilRequest();
    int fetchCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    // Now receive a BeginEpoch from `voter3`
    context.deliverRequest(context.beginEpochRequest(epoch + 1, voter3));
    context.client.poll();
    context.assertElectedLeader(epoch + 1, voter3);
    // The fetch response from the old leader returns, but it should be ignored
    Records records = context.buildBatch(0L, 3, Arrays.asList("a", "b"));
    FetchResponseData response = context.fetchResponse(epoch, voter2, records, 0L, Errors.NONE);
    context.deliverResponse(fetchCorrelationId, voter2, response);
    context.client.poll();
    assertEquals(0, context.log.endOffset().offset);
    context.assertElectedLeader(epoch + 1, voter3);
}

Example 13 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClientTest method testFollowerLogReconciliation.

@Test
public void testFollowerLogReconciliation() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    int lastEpoch = 3;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, otherNodeId).appendToLog(lastEpoch, Arrays.asList("foo", "bar")).appendToLog(lastEpoch, Arrays.asList("baz")).build();
    context.assertElectedLeader(epoch, otherNodeId);
    assertEquals(3L, context.log.endOffset().offset);
    context.pollUntilRequest();
    int correlationId = context.assertSentFetchRequest(epoch, 3L, lastEpoch);
    FetchResponseData response = context.divergingFetchResponse(epoch, otherNodeId, 2L, lastEpoch, 1L);
    context.deliverResponse(correlationId, otherNodeId, response);
    // Poll again to complete truncation
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    // Now we should be fetching
    context.client.poll();
    context.assertSentFetchRequest(epoch, 2L, lastEpoch);
}

Example 14 with FetchResponseData

use of org.apache.kafka.common.message.FetchResponseData in project kafka by apache.

the class KafkaRaftClient method handleFetchRequest.

/**
 * Handle a Fetch request. The fetch offset and last fetched epoch are always
 * validated against the current log. In the case that they do not match, the response will
 * indicate the diverging offset/epoch. A follower is expected to truncate its log in this
 * case and resend the fetch.
 *
 * This API may return the following errors:
 *
 * - {@link Errors#INCONSISTENT_CLUSTER_ID} if the cluster id is presented in request
 *     but different from this node
 * - {@link Errors#BROKER_NOT_AVAILABLE} if this node is currently shutting down
 * - {@link Errors#FENCED_LEADER_EPOCH} if the epoch is smaller than this node's epoch
 * - {@link Errors#INVALID_REQUEST} if the request epoch is larger than the leader's current epoch
 *     or if either the fetch offset or the last fetched epoch is invalid
 */
private CompletableFuture<FetchResponseData> handleFetchRequest(RaftRequest.Inbound requestMetadata, long currentTimeMs) {
    FetchRequestData request = (FetchRequestData) requestMetadata.data;
    if (!hasValidClusterId(request.clusterId())) {
        return completedFuture(new FetchResponseData().setErrorCode(Errors.INCONSISTENT_CLUSTER_ID.code()));
    }
    if (!hasValidTopicPartition(request, log.topicPartition(), log.topicId())) {
        // Until we support multi-raft, we treat topic partition mismatches as invalid requests
        return completedFuture(new FetchResponseData().setErrorCode(Errors.INVALID_REQUEST.code()));
    }
    // If the ID is valid, we can set the topic name.
    request.topics().get(0).setTopic(log.topicPartition().topic());
    FetchRequestData.FetchPartition fetchPartition = request.topics().get(0).partitions().get(0);
    if (request.maxWaitMs() < 0 || fetchPartition.fetchOffset() < 0 || fetchPartition.lastFetchedEpoch() < 0 || fetchPartition.lastFetchedEpoch() > fetchPartition.currentLeaderEpoch()) {
        return completedFuture(buildEmptyFetchResponse(Errors.INVALID_REQUEST, Optional.empty()));
    }
    FetchResponseData response = tryCompleteFetchRequest(request.replicaId(), fetchPartition, currentTimeMs);
    FetchResponseData.PartitionData partitionResponse = response.responses().get(0).partitions().get(0);
    if (partitionResponse.errorCode() != Errors.NONE.code() || FetchResponse.recordsSize(partitionResponse) > 0 || request.maxWaitMs() == 0) {
        return completedFuture(response);
    }
    CompletableFuture<Long> future = fetchPurgatory.await(fetchPartition.fetchOffset(), request.maxWaitMs());
    return future.handle((completionTimeMs, exception) -> {
        if (exception != null) {
            Throwable cause = exception instanceof ExecutionException ? exception.getCause() : exception;
            // If the fetch timed out in purgatory, it means no new data is available,
            // and we will complete the fetch successfully. Otherwise, if there was
            // any other error, we need to return it.
            Errors error = Errors.forException(cause);
            if (error != Errors.REQUEST_TIMED_OUT) {
                logger.debug("Failed to handle fetch from {} at {} due to {}", request.replicaId(), fetchPartition.fetchOffset(), error);
                return buildEmptyFetchResponse(error, Optional.empty());
            }
        }
        // FIXME: `completionTimeMs`, which can be null
        logger.trace("Completing delayed fetch from {} starting at offset {} at {}", request.replicaId(), fetchPartition.fetchOffset(), completionTimeMs);
        return tryCompleteFetchRequest(request.replicaId(), fetchPartition, time.milliseconds());
    });
}