Examples with TopicIdPartition org.apache.kafka.common.TopicIdPartition used on opensource projects

Example 21 with TopicIdPartition

use of org.apache.kafka.common.TopicIdPartition in project kafka by apache.

the class FetcherTest method testFetchWithNoTopicId.

@Test
public void testFetchWithNoTopicId() {
    // Should work and default to using old request type.
    buildFetcher();
    TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0));
    assignFromUserNoId(singleton(noId.topicPartition()));
    subscriptions.seek(noId.topicPartition(), 0);
    // Fetch should use request version 12
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(noId.topicPartition()));
    List<ConsumerRecord<byte[], byte[]>> records = partitionRecords.get(noId.topicPartition());
    assertEquals(3, records.size());
    // this is the next fetching position
    assertEquals(4L, subscriptions.position(noId.topicPartition()).offset);
    long offset = 1;
    for (ConsumerRecord<byte[], byte[]> record : records) {
        assertEquals(offset, record.offset());
        offset += 1;
    }
}

Example 22 with TopicIdPartition

use of org.apache.kafka.common.TopicIdPartition in project kafka by apache.

the class TopicBasedRemoteLogMetadataManagerTest method testNewPartitionUpdates.

@Test
public void testNewPartitionUpdates() throws Exception {
    // Create topics.
    String leaderTopic = "new-leader";
    HashMap<Object, Seq<Object>> assignedLeaderTopicReplicas = new HashMap<>();
    List<Object> leaderTopicReplicas = new ArrayList<>();
    // Set broker id 0 as the first entry which is taken as the leader.
    leaderTopicReplicas.add(0);
    leaderTopicReplicas.add(1);
    leaderTopicReplicas.add(2);
    assignedLeaderTopicReplicas.put(0, JavaConverters.asScalaBuffer(leaderTopicReplicas));
    remoteLogMetadataManagerHarness.createTopicWithAssignment(leaderTopic, JavaConverters.mapAsScalaMap(assignedLeaderTopicReplicas), remoteLogMetadataManagerHarness.listenerName());
    String followerTopic = "new-follower";
    HashMap<Object, Seq<Object>> assignedFollowerTopicReplicas = new HashMap<>();
    List<Object> followerTopicReplicas = new ArrayList<>();
    // Set broker id 1 as the first entry which is taken as the leader.
    followerTopicReplicas.add(1);
    followerTopicReplicas.add(2);
    followerTopicReplicas.add(0);
    assignedFollowerTopicReplicas.put(0, JavaConverters.asScalaBuffer(followerTopicReplicas));
    remoteLogMetadataManagerHarness.createTopicWithAssignment(followerTopic, JavaConverters.mapAsScalaMap(assignedFollowerTopicReplicas), remoteLogMetadataManagerHarness.listenerName());
    final TopicIdPartition newLeaderTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition(leaderTopic, 0));
    final TopicIdPartition newFollowerTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition(followerTopic, 0));
    // Add segments for these partitions but an exception is received as they have not yet been subscribed.
    // These messages would have been published to the respective metadata topic partitions but the ConsumerManager
    // has not yet been subscribing as they are not yet registered.
    RemoteLogSegmentMetadata leaderSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newLeaderTopicIdPartition, Uuid.randomUuid()), 0, 100, -1L, 0, time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
    Assertions.assertThrows(Exception.class, () -> topicBasedRlmm().addRemoteLogSegmentMetadata(leaderSegmentMetadata).get());
    RemoteLogSegmentMetadata followerSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newFollowerTopicIdPartition, Uuid.randomUuid()), 0, 100, -1L, 0, time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
    Assertions.assertThrows(Exception.class, () -> topicBasedRlmm().addRemoteLogSegmentMetadata(followerSegmentMetadata).get());
    // `listRemoteLogSegments` will receive an exception as these topic partitions are not yet registered.
    Assertions.assertThrows(RemoteResourceNotFoundException.class, () -> topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition));
    Assertions.assertThrows(RemoteResourceNotFoundException.class, () -> topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition));
    topicBasedRlmm().onPartitionLeadershipChanges(Collections.singleton(newLeaderTopicIdPartition), Collections.singleton(newFollowerTopicIdPartition));
    // RemoteLogSegmentMetadata events are already published, and topicBasedRlmm's consumer manager will start
    // fetching those events and build the cache.
    waitUntilConsumerCatchesup(newLeaderTopicIdPartition, newFollowerTopicIdPartition, 30_000L);
    Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition).hasNext());
    Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition).hasNext());
}

Example 23 with TopicIdPartition

use of org.apache.kafka.common.TopicIdPartition in project kafka by apache.

the class InmemoryRemoteStorageManagerTest method createRemoteLogSegmentMetadata.

private RemoteLogSegmentMetadata createRemoteLogSegmentMetadata() {
    TopicIdPartition topicPartition = new TopicIdPartition(Uuid.randomUuid(), TP);
    RemoteLogSegmentId id = new RemoteLogSegmentId(topicPartition, Uuid.randomUuid());
    return new RemoteLogSegmentMetadata(id, 100L, 200L, System.currentTimeMillis(), 0, System.currentTimeMillis(), 100, Collections.singletonMap(1, 100L));
}

Example 24 with TopicIdPartition

use of org.apache.kafka.common.TopicIdPartition in project kafka by apache.

the class RemotePartitionMetadataStore method handleRemoteLogSegmentMetadata.

@Override
public void handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata) {
    log.debug("Adding remote log segment : [{}]", remoteLogSegmentMetadata);
    final RemoteLogSegmentId remoteLogSegmentId = remoteLogSegmentMetadata.remoteLogSegmentId();
    TopicIdPartition topicIdPartition = remoteLogSegmentId.topicIdPartition();
    // This should have been already existing as it is loaded when the partitions are assigned.
    RemoteLogMetadataCache remoteLogMetadataCache = idToRemoteLogMetadataCache.get(topicIdPartition);
    if (remoteLogMetadataCache != null) {
        remoteLogMetadataCache.addCopyInProgressSegment(remoteLogSegmentMetadata);
    } else {
        throw new IllegalStateException("No partition metadata found for : " + topicIdPartition);
    }
}

Example 25 with TopicIdPartition

use of org.apache.kafka.common.TopicIdPartition in project kafka by apache.

the class RemotePartitionMetadataStore method handleRemoteLogSegmentMetadataUpdate.

@Override
public void handleRemoteLogSegmentMetadataUpdate(RemoteLogSegmentMetadataUpdate rlsmUpdate) {
    log.debug("Updating remote log segment: [{}]", rlsmUpdate);
    RemoteLogSegmentId remoteLogSegmentId = rlsmUpdate.remoteLogSegmentId();
    TopicIdPartition topicIdPartition = remoteLogSegmentId.topicIdPartition();
    RemoteLogMetadataCache remoteLogMetadataCache = idToRemoteLogMetadataCache.get(topicIdPartition);
    if (remoteLogMetadataCache != null) {
        try {
            remoteLogMetadataCache.updateRemoteLogSegmentMetadata(rlsmUpdate);
        } catch (RemoteResourceNotFoundException e) {
            log.warn("Error occurred while updating the remote log segment.", e);
        }
    } else {
        throw new IllegalStateException("No partition metadata found for : " + topicIdPartition);
    }
}