Examples with Records org.apache.kafka.common.record.Records used on opensource projects

Example 6 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class MockLogTest method testAppendControlRecord.

@Test
public void testAppendControlRecord() {
    final long initialOffset = 0;
    final int currentEpoch = 3;
    LeaderChangeMessage messageData = new LeaderChangeMessage().setLeaderId(0);
    ByteBuffer buffer = ByteBuffer.allocate(256);
    log.appendAsLeader(MemoryRecords.withLeaderChangeMessage(initialOffset, 0L, 2, buffer, messageData), currentEpoch);
    assertEquals(0, log.startOffset());
    assertEquals(1, log.endOffset().offset);
    assertEquals(currentEpoch, log.lastFetchedEpoch());
    Records records = log.read(0, Isolation.UNCOMMITTED).records;
    for (RecordBatch batch : records.batches()) {
        assertTrue(batch.isControlBatch());
    }
    List<ByteBuffer> extractRecords = new ArrayList<>();
    for (Record record : records.records()) {
        LeaderChangeMessage deserializedData = ControlRecordUtils.deserializeLeaderChangeMessage(record);
        assertEquals(deserializedData, messageData);
        extractRecords.add(record.value());
    }
    assertEquals(1, extractRecords.size());
    assertEquals(new OffsetAndEpoch(1, currentEpoch), log.endOffsetForEpoch(currentEpoch));
}

Example 7 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class KafkaRaftClient method handleFetchResponse.

private boolean handleFetchResponse(RaftResponse.Inbound responseMetadata, long currentTimeMs) {
    FetchResponseData response = (FetchResponseData) responseMetadata.data;
    Errors topLevelError = Errors.forCode(response.errorCode());
    if (topLevelError != Errors.NONE) {
        return handleTopLevelError(topLevelError, responseMetadata);
    }
    if (!RaftUtil.hasValidTopicPartition(response, log.topicPartition(), log.topicId())) {
        return false;
    }
    // If the ID is valid, we can set the topic name.
    response.responses().get(0).setTopic(log.topicPartition().topic());
    FetchResponseData.PartitionData partitionResponse = response.responses().get(0).partitions().get(0);
    FetchResponseData.LeaderIdAndEpoch currentLeaderIdAndEpoch = partitionResponse.currentLeader();
    OptionalInt responseLeaderId = optionalLeaderId(currentLeaderIdAndEpoch.leaderId());
    int responseEpoch = currentLeaderIdAndEpoch.leaderEpoch();
    Errors error = Errors.forCode(partitionResponse.errorCode());
    Optional<Boolean> handled = maybeHandleCommonResponse(error, responseLeaderId, responseEpoch, currentTimeMs);
    if (handled.isPresent()) {
        return handled.get();
    }
    FollowerState state = quorum.followerStateOrThrow();
    if (error == Errors.NONE) {
        FetchResponseData.EpochEndOffset divergingEpoch = partitionResponse.divergingEpoch();
        if (divergingEpoch.epoch() >= 0) {
            // The leader is asking us to truncate before continuing
            final OffsetAndEpoch divergingOffsetAndEpoch = new OffsetAndEpoch(divergingEpoch.endOffset(), divergingEpoch.epoch());
            state.highWatermark().ifPresent(highWatermark -> {
                if (divergingOffsetAndEpoch.offset < highWatermark.offset) {
                    throw new KafkaException("The leader requested truncation to offset " + divergingOffsetAndEpoch.offset + ", which is below the current high watermark" + " " + highWatermark);
                }
            });
            long truncationOffset = log.truncateToEndOffset(divergingOffsetAndEpoch);
            logger.info("Truncated to offset {} from Fetch response from leader {}", truncationOffset, quorum.leaderIdOrSentinel());
        } else if (partitionResponse.snapshotId().epoch() >= 0 || partitionResponse.snapshotId().endOffset() >= 0) {
            if (partitionResponse.snapshotId().epoch() < 0) {
                logger.error("The leader sent a snapshot id with a valid end offset {} but with an invalid epoch {}", partitionResponse.snapshotId().endOffset(), partitionResponse.snapshotId().epoch());
                return false;
            } else if (partitionResponse.snapshotId().endOffset() < 0) {
                logger.error("The leader sent a snapshot id with a valid epoch {} but with an invalid end offset {}", partitionResponse.snapshotId().epoch(), partitionResponse.snapshotId().endOffset());
                return false;
            } else {
                final OffsetAndEpoch snapshotId = new OffsetAndEpoch(partitionResponse.snapshotId().endOffset(), partitionResponse.snapshotId().epoch());
                // Do not validate the snapshot id against the local replicated log
                // since this snapshot is expected to reference offsets and epochs
                // greater than the log end offset and high-watermark
                state.setFetchingSnapshot(log.storeSnapshot(snapshotId));
            }
        } else {
            Records records = FetchResponse.recordsOrFail(partitionResponse);
            if (records.sizeInBytes() > 0) {
                appendAsFollower(records);
            }
            OptionalLong highWatermark = partitionResponse.highWatermark() < 0 ? OptionalLong.empty() : OptionalLong.of(partitionResponse.highWatermark());
            updateFollowerHighWatermark(state, highWatermark);
        }
        state.resetFetchTimeout(currentTimeMs);
        return true;
    } else {
        return handleUnexpectedError(error, responseMetadata);
    }
}

Example 8 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class KafkaRaftClient method tryCompleteFetchRequest.

private FetchResponseData tryCompleteFetchRequest(int replicaId, FetchRequestData.FetchPartition request, long currentTimeMs) {
    try {
        Optional<Errors> errorOpt = validateLeaderOnlyRequest(request.currentLeaderEpoch());
        if (errorOpt.isPresent()) {
            return buildEmptyFetchResponse(errorOpt.get(), Optional.empty());
        }
        long fetchOffset = request.fetchOffset();
        int lastFetchedEpoch = request.lastFetchedEpoch();
        LeaderState<T> state = quorum.leaderStateOrThrow();
        ValidOffsetAndEpoch validOffsetAndEpoch = log.validateOffsetAndEpoch(fetchOffset, lastFetchedEpoch);
        final Records records;
        if (validOffsetAndEpoch.kind() == ValidOffsetAndEpoch.Kind.VALID) {
            LogFetchInfo info = log.read(fetchOffset, Isolation.UNCOMMITTED);
            if (state.updateReplicaState(replicaId, currentTimeMs, info.startOffsetMetadata)) {
                onUpdateLeaderHighWatermark(state, currentTimeMs);
            }
            records = info.records;
        } else {
            records = MemoryRecords.EMPTY;
        }
        return buildFetchResponse(Errors.NONE, records, validOffsetAndEpoch, state.highWatermark());
    } catch (Exception e) {
        logger.error("Caught unexpected error in fetch completion of request {}", request, e);
        return buildEmptyFetchResponse(Errors.UNKNOWN_SERVER_ERROR, Optional.empty());
    }
}

Example 9 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class KafkaRaftClientTest method testFollowerReplication.

@Test
public void testFollowerReplication() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, otherNodeId).build();
    context.assertElectedLeader(epoch, otherNodeId);
    context.pollUntilRequest();
    int fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    Records records = context.buildBatch(0L, 3, Arrays.asList("a", "b"));
    FetchResponseData response = context.fetchResponse(epoch, otherNodeId, records, 0L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, response);
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    assertEquals(2L, context.log.lastFlushedOffset());
}

Example 10 with Records

use of org.apache.kafka.common.record.Records in project kafka by apache.

the class KafkaRaftClientTest method testEmptyRecordSetInFetchResponse.

@Test
public void testEmptyRecordSetInFetchResponse() throws Exception {
    int localId = 0;
    int otherNodeId = 1;
    int epoch = 5;
    Set<Integer> voters = Utils.mkSet(localId, otherNodeId);
    RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).withElectedLeader(epoch, otherNodeId).build();
    context.assertElectedLeader(epoch, otherNodeId);
    // Receive an empty fetch response
    context.pollUntilRequest();
    int fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    FetchResponseData fetchResponse = context.fetchResponse(epoch, otherNodeId, MemoryRecords.EMPTY, 0L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, fetchResponse);
    context.client.poll();
    assertEquals(0L, context.log.endOffset().offset);
    assertEquals(OptionalLong.of(0L), context.client.highWatermark());
    // Receive some records in the next poll, but do not advance high watermark
    context.pollUntilRequest();
    Records records = context.buildBatch(0L, epoch, Arrays.asList("a", "b"));
    fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 0L, 0);
    fetchResponse = context.fetchResponse(epoch, otherNodeId, records, 0L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, fetchResponse);
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    assertEquals(OptionalLong.of(0L), context.client.highWatermark());
    // The next fetch response is empty, but should still advance the high watermark
    context.pollUntilRequest();
    fetchQuorumCorrelationId = context.assertSentFetchRequest(epoch, 2L, epoch);
    fetchResponse = context.fetchResponse(epoch, otherNodeId, MemoryRecords.EMPTY, 2L, Errors.NONE);
    context.deliverResponse(fetchQuorumCorrelationId, otherNodeId, fetchResponse);
    context.client.poll();
    assertEquals(2L, context.log.endOffset().offset);
    assertEquals(OptionalLong.of(2L), context.client.highWatermark());
}