Examples with RemoteLogSegmentMetadata org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata used on opensource projects

Example 1 with RemoteLogSegmentMetadata

use of org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata in project kafka by apache.

the class TopicBasedRemoteLogMetadataManagerTest method testNewPartitionUpdates.

@Test
public void testNewPartitionUpdates() throws Exception {
    // Create topics.
    String leaderTopic = "new-leader";
    HashMap<Object, Seq<Object>> assignedLeaderTopicReplicas = new HashMap<>();
    List<Object> leaderTopicReplicas = new ArrayList<>();
    // Set broker id 0 as the first entry which is taken as the leader.
    leaderTopicReplicas.add(0);
    leaderTopicReplicas.add(1);
    leaderTopicReplicas.add(2);
    assignedLeaderTopicReplicas.put(0, JavaConverters.asScalaBuffer(leaderTopicReplicas));
    remoteLogMetadataManagerHarness.createTopicWithAssignment(leaderTopic, JavaConverters.mapAsScalaMap(assignedLeaderTopicReplicas), remoteLogMetadataManagerHarness.listenerName());
    String followerTopic = "new-follower";
    HashMap<Object, Seq<Object>> assignedFollowerTopicReplicas = new HashMap<>();
    List<Object> followerTopicReplicas = new ArrayList<>();
    // Set broker id 1 as the first entry which is taken as the leader.
    followerTopicReplicas.add(1);
    followerTopicReplicas.add(2);
    followerTopicReplicas.add(0);
    assignedFollowerTopicReplicas.put(0, JavaConverters.asScalaBuffer(followerTopicReplicas));
    remoteLogMetadataManagerHarness.createTopicWithAssignment(followerTopic, JavaConverters.mapAsScalaMap(assignedFollowerTopicReplicas), remoteLogMetadataManagerHarness.listenerName());
    final TopicIdPartition newLeaderTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition(leaderTopic, 0));
    final TopicIdPartition newFollowerTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition(followerTopic, 0));
    // Add segments for these partitions but an exception is received as they have not yet been subscribed.
    // These messages would have been published to the respective metadata topic partitions but the ConsumerManager
    // has not yet been subscribing as they are not yet registered.
    RemoteLogSegmentMetadata leaderSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newLeaderTopicIdPartition, Uuid.randomUuid()), 0, 100, -1L, 0, time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
    Assertions.assertThrows(Exception.class, () -> topicBasedRlmm().addRemoteLogSegmentMetadata(leaderSegmentMetadata).get());
    RemoteLogSegmentMetadata followerSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newFollowerTopicIdPartition, Uuid.randomUuid()), 0, 100, -1L, 0, time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
    Assertions.assertThrows(Exception.class, () -> topicBasedRlmm().addRemoteLogSegmentMetadata(followerSegmentMetadata).get());
    // `listRemoteLogSegments` will receive an exception as these topic partitions are not yet registered.
    Assertions.assertThrows(RemoteResourceNotFoundException.class, () -> topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition));
    Assertions.assertThrows(RemoteResourceNotFoundException.class, () -> topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition));
    topicBasedRlmm().onPartitionLeadershipChanges(Collections.singleton(newLeaderTopicIdPartition), Collections.singleton(newFollowerTopicIdPartition));
    // RemoteLogSegmentMetadata events are already published, and topicBasedRlmm's consumer manager will start
    // fetching those events and build the cache.
    waitUntilConsumerCatchesup(newLeaderTopicIdPartition, newFollowerTopicIdPartition, 30_000L);
    Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition).hasNext());
    Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition).hasNext());
}

Example 2 with RemoteLogSegmentMetadata

use of org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata in project kafka by apache.

the class RemoteLogSegmentLifecycleTest method testCacheListSegments.

@ParameterizedTest(name = "remoteLogSegmentLifecycleManager = {0}")
@MethodSource("remoteLogSegmentLifecycleManagers")
public void testCacheListSegments(RemoteLogSegmentLifecycleManager remoteLogSegmentLifecycleManager) throws Exception {
    try {
        remoteLogSegmentLifecycleManager.initialize(topicIdPartition);
        // Create a few segments and add them to the cache.
        RemoteLogSegmentMetadata segment0 = createSegmentUpdateWithState(remoteLogSegmentLifecycleManager, Collections.singletonMap(0, 0L), 0, 100, RemoteLogSegmentState.COPY_SEGMENT_FINISHED);
        RemoteLogSegmentMetadata segment1 = createSegmentUpdateWithState(remoteLogSegmentLifecycleManager, Collections.singletonMap(0, 101L), 101, 200, RemoteLogSegmentState.COPY_SEGMENT_FINISHED);
        Map<Integer, Long> segment2LeaderEpochs = new HashMap<>();
        segment2LeaderEpochs.put(0, 201L);
        segment2LeaderEpochs.put(1, 301L);
        RemoteLogSegmentMetadata segment2 = createSegmentUpdateWithState(remoteLogSegmentLifecycleManager, segment2LeaderEpochs, 201, 400, RemoteLogSegmentState.COPY_SEGMENT_FINISHED);
        // listRemoteLogSegments(0) and listAllRemoteLogSegments() should contain all the above segments.
        List<RemoteLogSegmentMetadata> expectedSegmentsForEpoch0 = Arrays.asList(segment0, segment1, segment2);
        Assertions.assertTrue(TestUtils.sameElementsWithOrder(remoteLogSegmentLifecycleManager.listRemoteLogSegments(0), expectedSegmentsForEpoch0.iterator()));
        Assertions.assertTrue(TestUtils.sameElementsWithoutOrder(remoteLogSegmentLifecycleManager.listAllRemoteLogSegments(), expectedSegmentsForEpoch0.iterator()));
        // listRemoteLogSegments(1) should contain only segment2.
        List<RemoteLogSegmentMetadata> expectedSegmentsForEpoch1 = Collections.singletonList(segment2);
        Assertions.assertTrue(TestUtils.sameElementsWithOrder(remoteLogSegmentLifecycleManager.listRemoteLogSegments(1), expectedSegmentsForEpoch1.iterator()));
    } finally {
        Utils.closeQuietly(remoteLogSegmentLifecycleManager, "RemoteLogSegmentLifecycleManager");
    }
}

Example 3 with RemoteLogSegmentMetadata

use of org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata in project kafka by apache.

the class RemoteLogSegmentLifecycleTest method testRemoteLogSegmentLifeCycle.

@ParameterizedTest(name = "remoteLogSegmentLifecycleManager = {0}")
@MethodSource("remoteLogSegmentLifecycleManagers")
public void testRemoteLogSegmentLifeCycle(RemoteLogSegmentLifecycleManager remoteLogSegmentLifecycleManager) throws Exception {
    try {
        remoteLogSegmentLifecycleManager.initialize(topicIdPartition);
        // segment 0
        // offsets: [0-100]
        // leader epochs (0,0), (1,20), (2,80)
        Map<Integer, Long> segment0LeaderEpochs = new HashMap<>();
        segment0LeaderEpochs.put(0, 0L);
        segment0LeaderEpochs.put(1, 20L);
        segment0LeaderEpochs.put(2, 80L);
        RemoteLogSegmentId segment0Id = new RemoteLogSegmentId(topicIdPartition, Uuid.randomUuid());
        RemoteLogSegmentMetadata segment0Metadata = new RemoteLogSegmentMetadata(segment0Id, 0L, 100L, -1L, BROKER_ID_0, time.milliseconds(), SEG_SIZE, segment0LeaderEpochs);
        remoteLogSegmentLifecycleManager.addRemoteLogSegmentMetadata(segment0Metadata);
        // We should not get this as the segment is still getting copied and it is not yet considered successful until
        // it reaches RemoteLogSegmentState.COPY_SEGMENT_FINISHED.
        Assertions.assertFalse(remoteLogSegmentLifecycleManager.remoteLogSegmentMetadata(40, 1).isPresent());
        // Check that these leader epochs are not to be considered for highestOffsetForEpoch API as they are still getting copied.
        Stream.of(0, 1, 2).forEach(epoch -> {
            try {
                Assertions.assertFalse(remoteLogSegmentLifecycleManager.highestOffsetForEpoch(epoch).isPresent());
            } catch (RemoteStorageException e) {
                Assertions.fail(e);
            }
        });
        RemoteLogSegmentMetadataUpdate segment0Update = new RemoteLogSegmentMetadataUpdate(segment0Id, time.milliseconds(), RemoteLogSegmentState.COPY_SEGMENT_FINISHED, BROKER_ID_1);
        remoteLogSegmentLifecycleManager.updateRemoteLogSegmentMetadata(segment0Update);
        RemoteLogSegmentMetadata expectedSegment0Metadata = segment0Metadata.createWithUpdates(segment0Update);
        // segment 1
        // offsets: [101 - 200]
        // no changes in leadership with in this segment
        // leader epochs (2, 101)
        Map<Integer, Long> segment1LeaderEpochs = Collections.singletonMap(2, 101L);
        RemoteLogSegmentMetadata segment1Metadata = createSegmentUpdateWithState(remoteLogSegmentLifecycleManager, segment1LeaderEpochs, 101L, 200L, RemoteLogSegmentState.COPY_SEGMENT_FINISHED);
        // segment 2
        // offsets: [201 - 300]
        // moved to epoch 3 in between
        // leader epochs (2, 201), (3, 240)
        Map<Integer, Long> segment2LeaderEpochs = new HashMap<>();
        segment2LeaderEpochs.put(2, 201L);
        segment2LeaderEpochs.put(3, 240L);
        RemoteLogSegmentMetadata segment2Metadata = createSegmentUpdateWithState(remoteLogSegmentLifecycleManager, segment2LeaderEpochs, 201L, 300L, RemoteLogSegmentState.COPY_SEGMENT_FINISHED);
        // segment 3
        // offsets: [250 - 400]
        // leader epochs (3, 250), (4, 370)
        Map<Integer, Long> segment3LeaderEpochs = new HashMap<>();
        segment3LeaderEpochs.put(3, 250L);
        segment3LeaderEpochs.put(4, 370L);
        RemoteLogSegmentMetadata segment3Metadata = createSegmentUpdateWithState(remoteLogSegmentLifecycleManager, segment3LeaderEpochs, 250L, 400L, RemoteLogSegmentState.COPY_SEGMENT_FINISHED);
        // ////////////////////////////////////////////////////////////////////////////////////////
        // Four segments are added with different boundaries and leader epochs.
        // Search for cache.remoteLogSegmentMetadata(leaderEpoch, offset)  for different
        // epochs and offsets
        // ////////////////////////////////////////////////////////////////////////////////////////
        HashMap<EpochOffset, RemoteLogSegmentMetadata> expectedEpochOffsetToSegmentMetadata = new HashMap<>();
        // Existing metadata entries.
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(1, 40), expectedSegment0Metadata);
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(2, 110), segment1Metadata);
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(3, 240), segment2Metadata);
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(3, 250), segment3Metadata);
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(4, 375), segment3Metadata);
        // Non existing metadata entries.
        // Search for offset 110, epoch 1, and it should not exist.
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(1, 110), null);
        // Search for non existing offset 401, epoch 4.
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(4, 401), null);
        // Search for non existing epoch 5.
        expectedEpochOffsetToSegmentMetadata.put(new EpochOffset(5, 301), null);
        for (Map.Entry<EpochOffset, RemoteLogSegmentMetadata> entry : expectedEpochOffsetToSegmentMetadata.entrySet()) {
            EpochOffset epochOffset = entry.getKey();
            Optional<RemoteLogSegmentMetadata> segmentMetadata = remoteLogSegmentLifecycleManager.remoteLogSegmentMetadata(epochOffset.epoch, epochOffset.offset);
            RemoteLogSegmentMetadata expectedSegmentMetadata = entry.getValue();
            log.debug("Searching for {} , result: {}, expected: {} ", epochOffset, segmentMetadata, expectedSegmentMetadata);
            if (expectedSegmentMetadata != null) {
                Assertions.assertEquals(Optional.of(expectedSegmentMetadata), segmentMetadata);
            } else {
                Assertions.assertFalse(segmentMetadata.isPresent());
            }
        }
        // Update segment with state as DELETE_SEGMENT_STARTED.
        // It should not be available when we search for that segment.
        remoteLogSegmentLifecycleManager.updateRemoteLogSegmentMetadata(new RemoteLogSegmentMetadataUpdate(expectedSegment0Metadata.remoteLogSegmentId(), time.milliseconds(), RemoteLogSegmentState.DELETE_SEGMENT_STARTED, BROKER_ID_1));
        Assertions.assertFalse(remoteLogSegmentLifecycleManager.remoteLogSegmentMetadata(0, 10).isPresent());
        // Update segment with state as DELETE_SEGMENT_FINISHED.
        // It should not be available when we search for that segment.
        remoteLogSegmentLifecycleManager.updateRemoteLogSegmentMetadata(new RemoteLogSegmentMetadataUpdate(expectedSegment0Metadata.remoteLogSegmentId(), time.milliseconds(), RemoteLogSegmentState.DELETE_SEGMENT_FINISHED, BROKER_ID_1));
        Assertions.assertFalse(remoteLogSegmentLifecycleManager.remoteLogSegmentMetadata(0, 10).isPresent());
        // ////////////////////////////////////////////////////////////////////////////////////////
        // Search for cache.highestLogOffset(leaderEpoch) for all the leader epochs
        // ////////////////////////////////////////////////////////////////////////////////////////
        Map<Integer, Long> expectedEpochToHighestOffset = new HashMap<>();
        expectedEpochToHighestOffset.put(0, 19L);
        expectedEpochToHighestOffset.put(1, 79L);
        expectedEpochToHighestOffset.put(2, 239L);
        expectedEpochToHighestOffset.put(3, 369L);
        expectedEpochToHighestOffset.put(4, 400L);
        for (Map.Entry<Integer, Long> entry : expectedEpochToHighestOffset.entrySet()) {
            Integer epoch = entry.getKey();
            Long expectedOffset = entry.getValue();
            Optional<Long> offset = remoteLogSegmentLifecycleManager.highestOffsetForEpoch(epoch);
            log.debug("Fetching highest offset for epoch: {} , returned: {} , expected: {}", epoch, offset, expectedOffset);
            Assertions.assertEquals(Optional.of(expectedOffset), offset);
        }
        // Search for non existing leader epoch
        Optional<Long> highestOffsetForEpoch5 = remoteLogSegmentLifecycleManager.highestOffsetForEpoch(5);
        Assertions.assertFalse(highestOffsetForEpoch5.isPresent());
    } finally {
        Utils.closeQuietly(remoteLogSegmentLifecycleManager, "RemoteLogSegmentLifecycleManager");
    }
}

Example 4 with RemoteLogSegmentMetadata

use of org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata in project kafka by apache.

the class RemoteLogMetadataCache method remoteLogSegmentMetadata.

/**
 * Returns {@link RemoteLogSegmentMetadata} if it exists for the given leader-epoch containing the offset and with
 * {@link RemoteLogSegmentState#COPY_SEGMENT_FINISHED} state, else returns {@link Optional#empty()}.
 *
 * @param leaderEpoch leader epoch for the given offset
 * @param offset      offset
 * @return the requested remote log segment metadata if it exists.
 */
public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(int leaderEpoch, long offset) {
    RemoteLogLeaderEpochState remoteLogLeaderEpochState = leaderEpochEntries.get(leaderEpoch);
    if (remoteLogLeaderEpochState == null) {
        return Optional.empty();
    }
    // Look for floor entry as the given offset may exist in this entry.
    RemoteLogSegmentId remoteLogSegmentId = remoteLogLeaderEpochState.floorEntry(offset);
    if (remoteLogSegmentId == null) {
        // If the offset is lower than the minimum offset available in metadata then return empty.
        return Optional.empty();
    }
    RemoteLogSegmentMetadata metadata = idToSegmentMetadata.get(remoteLogSegmentId);
    // Check whether the given offset with leaderEpoch exists in this segment.
    // Check for epoch's offset boundaries with in this segment.
    // 1. Get the next epoch's start offset -1 if exists
    // 2. If no next epoch exists, then segment end offset can be considered as epoch's relative end offset.
    Map.Entry<Integer, Long> nextEntry = metadata.segmentLeaderEpochs().higherEntry(leaderEpoch);
    long epochEndOffset = (nextEntry != null) ? nextEntry.getValue() - 1 : metadata.endOffset();
    // Return empty when target offset > epoch's end offset.
    return offset > epochEndOffset ? Optional.empty() : Optional.of(metadata);
}

Example 5 with RemoteLogSegmentMetadata

use of org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata in project kafka by apache.

the class RemoteLogMetadataCache method updateRemoteLogSegmentMetadata.

public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate metadataUpdate) throws RemoteResourceNotFoundException {
    log.debug("Updating remote log segment metadata: [{}]", metadataUpdate);
    Objects.requireNonNull(metadataUpdate, "metadataUpdate can not be null");
    RemoteLogSegmentState targetState = metadataUpdate.state();
    RemoteLogSegmentId remoteLogSegmentId = metadataUpdate.remoteLogSegmentId();
    RemoteLogSegmentMetadata existingMetadata = idToSegmentMetadata.get(remoteLogSegmentId);
    if (existingMetadata == null) {
        throw new RemoteResourceNotFoundException("No remote log segment metadata found for :" + remoteLogSegmentId);
    }
    // Check the state transition.
    checkStateTransition(existingMetadata.state(), targetState);
    switch(targetState) {
        case COPY_SEGMENT_STARTED:
            // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
            throw new IllegalArgumentException("metadataUpdate: " + metadataUpdate + " with state " + RemoteLogSegmentState.COPY_SEGMENT_STARTED + " can not be updated");
        case COPY_SEGMENT_FINISHED:
            handleSegmentWithCopySegmentFinishedState(existingMetadata.createWithUpdates(metadataUpdate));
            break;
        case DELETE_SEGMENT_STARTED:
            handleSegmentWithDeleteSegmentStartedState(existingMetadata.createWithUpdates(metadataUpdate));
            break;
        case DELETE_SEGMENT_FINISHED:
            handleSegmentWithDeleteSegmentFinishedState(existingMetadata.createWithUpdates(metadataUpdate));
            break;
        default:
            throw new IllegalArgumentException("Metadata with the state " + targetState + " is not supported");
    }
}