Examples with RawSnapshotWriter org.apache.kafka.snapshot.RawSnapshotWriter used on opensource projects

Example 16 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testFailToIncreaseLogStartPastHighWatermark.

@Test
public void testFailToIncreaseLogStartPastHighWatermark() throws IOException {
    int offset = 10;
    int epoch = 0;
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(2 * offset, epoch);
    appendBatch(3 * offset, epoch);
    log.updateHighWatermark(new LogOffsetMetadata(offset));
    try (RawSnapshotWriter snapshot = log.storeSnapshot(snapshotId).get()) {
        snapshot.freeze();
    }
    assertThrows(OffsetOutOfRangeException.class, () -> log.deleteBeforeSnapshot(snapshotId));
}

Example 17 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class MockLogTest method testTruncateWillRemoveOlderSnapshot.

@Test
public void testTruncateWillRemoveOlderSnapshot() throws IOException {
    int numberOfRecords = 10;
    int epoch = 1;
    OffsetAndEpoch sameEpochSnapshotId = new OffsetAndEpoch(numberOfRecords, epoch);
    appendBatch(numberOfRecords, epoch);
    log.updateHighWatermark(new LogOffsetMetadata(sameEpochSnapshotId.offset));
    try (RawSnapshotWriter snapshot = log.createNewSnapshot(sameEpochSnapshotId).get()) {
        snapshot.freeze();
    }
    OffsetAndEpoch greaterEpochSnapshotId = new OffsetAndEpoch(2 * numberOfRecords, epoch + 1);
    appendBatch(numberOfRecords, epoch);
    try (RawSnapshotWriter snapshot = log.storeSnapshot(greaterEpochSnapshotId).get()) {
        snapshot.freeze();
    }
    assertTrue(log.truncateToLatestSnapshot());
    assertEquals(Optional.empty(), log.readSnapshot(sameEpochSnapshotId));
}

Example 18 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class KafkaRaftClient method handleFetchSnapshotResponse.

private boolean handleFetchSnapshotResponse(RaftResponse.Inbound responseMetadata, long currentTimeMs) {
    FetchSnapshotResponseData data = (FetchSnapshotResponseData) responseMetadata.data;
    Errors topLevelError = Errors.forCode(data.errorCode());
    if (topLevelError != Errors.NONE) {
        return handleTopLevelError(topLevelError, responseMetadata);
    }
    if (data.topics().size() != 1 && data.topics().get(0).partitions().size() != 1) {
        return false;
    }
    Optional<FetchSnapshotResponseData.PartitionSnapshot> partitionSnapshotOpt = FetchSnapshotResponse.forTopicPartition(data, log.topicPartition());
    if (!partitionSnapshotOpt.isPresent()) {
        return false;
    }
    FetchSnapshotResponseData.PartitionSnapshot partitionSnapshot = partitionSnapshotOpt.get();
    FetchSnapshotResponseData.LeaderIdAndEpoch currentLeaderIdAndEpoch = partitionSnapshot.currentLeader();
    OptionalInt responseLeaderId = optionalLeaderId(currentLeaderIdAndEpoch.leaderId());
    int responseEpoch = currentLeaderIdAndEpoch.leaderEpoch();
    Errors error = Errors.forCode(partitionSnapshot.errorCode());
    Optional<Boolean> handled = maybeHandleCommonResponse(error, responseLeaderId, responseEpoch, currentTimeMs);
    if (handled.isPresent()) {
        return handled.get();
    }
    FollowerState state = quorum.followerStateOrThrow();
    if (Errors.forCode(partitionSnapshot.errorCode()) == Errors.SNAPSHOT_NOT_FOUND || partitionSnapshot.snapshotId().endOffset() < 0 || partitionSnapshot.snapshotId().epoch() < 0) {
        /* The leader deleted the snapshot before the follower could download it. Start over by
             * reseting the fetching snapshot state and sending another fetch request.
             */
        logger.trace("Leader doesn't know about snapshot id {}, returned error {} and snapshot id {}", state.fetchingSnapshot(), partitionSnapshot.errorCode(), partitionSnapshot.snapshotId());
        state.setFetchingSnapshot(Optional.empty());
        state.resetFetchTimeout(currentTimeMs);
        return true;
    }
    OffsetAndEpoch snapshotId = new OffsetAndEpoch(partitionSnapshot.snapshotId().endOffset(), partitionSnapshot.snapshotId().epoch());
    RawSnapshotWriter snapshot;
    if (state.fetchingSnapshot().isPresent()) {
        snapshot = state.fetchingSnapshot().get();
    } else {
        throw new IllegalStateException(String.format("Received unexpected fetch snapshot response: %s", partitionSnapshot));
    }
    if (!snapshot.snapshotId().equals(snapshotId)) {
        throw new IllegalStateException(String.format("Received fetch snapshot response with an invalid id. Expected %s; Received %s", snapshot.snapshotId(), snapshotId));
    }
    if (snapshot.sizeInBytes() != partitionSnapshot.position()) {
        throw new IllegalStateException(String.format("Received fetch snapshot response with an invalid position. Expected %s; Received %s", snapshot.sizeInBytes(), partitionSnapshot.position()));
    }
    final UnalignedMemoryRecords records;
    if (partitionSnapshot.unalignedRecords() instanceof MemoryRecords) {
        records = new UnalignedMemoryRecords(((MemoryRecords) partitionSnapshot.unalignedRecords()).buffer());
    } else if (partitionSnapshot.unalignedRecords() instanceof UnalignedMemoryRecords) {
        records = (UnalignedMemoryRecords) partitionSnapshot.unalignedRecords();
    } else {
        throw new IllegalStateException(String.format("Received unexpected fetch snapshot response: %s", partitionSnapshot));
    }
    snapshot.append(records);
    if (snapshot.sizeInBytes() == partitionSnapshot.size()) {
        // Finished fetching the snapshot.
        snapshot.freeze();
        state.setFetchingSnapshot(Optional.empty());
        if (log.truncateToLatestSnapshot()) {
            updateFollowerHighWatermark(state, OptionalLong.of(log.highWatermark().offset));
        } else {
            throw new IllegalStateException(String.format("Full log truncation expected but didn't happen. Snapshot of %s, log end offset %s, last fetched %s", snapshot.snapshotId(), log.endOffset(), log.lastFetchedEpoch()));
        }
    }
    state.resetFetchTimeout(currentTimeMs);
    return true;
}

Example 19 with RawSnapshotWriter

use of org.apache.kafka.snapshot.RawSnapshotWriter in project kafka by apache.

the class KafkaRaftClient method maybeSendFetchOrFetchSnapshot.

private long maybeSendFetchOrFetchSnapshot(FollowerState state, long currentTimeMs) {
    final Supplier<ApiMessage> requestSupplier;
    if (state.fetchingSnapshot().isPresent()) {
        RawSnapshotWriter snapshot = state.fetchingSnapshot().get();
        long snapshotSize = snapshot.sizeInBytes();
        requestSupplier = () -> buildFetchSnapshotRequest(snapshot.snapshotId(), snapshotSize);
    } else {
        requestSupplier = this::buildFetchRequest;
    }
    return maybeSendRequest(currentTimeMs, state.leaderId(), requestSupplier);
}