Search in sources :

Example 1 with TopicRecord

use of org.apache.kafka.common.metadata.TopicRecord in project kafka by apache.

the class TopicImage method write.

public void write(Consumer<List<ApiMessageAndVersion>> out) {
    List<ApiMessageAndVersion> batch = new ArrayList<>();
    batch.add(new ApiMessageAndVersion(new TopicRecord().setName(name).setTopicId(id), TOPIC_RECORD.highestSupportedVersion()));
    for (Entry<Integer, PartitionRegistration> entry : partitions.entrySet()) {
        int partitionId = entry.getKey();
        PartitionRegistration partition = entry.getValue();
        batch.add(partition.toRecord(id, partitionId));
    }
    out.accept(batch);
}
Also used : PartitionRegistration(org.apache.kafka.metadata.PartitionRegistration) TopicRecord(org.apache.kafka.common.metadata.TopicRecord) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) ArrayList(java.util.ArrayList)

Example 2 with TopicRecord

use of org.apache.kafka.common.metadata.TopicRecord in project kafka by apache.

the class MetadataNodeManagerTest method testTopicRecordAndRemoveTopicRecord.

@Test
public void testTopicRecordAndRemoveTopicRecord() {
    // Add topic
    TopicRecord topicRecord = new TopicRecord().setName("topicName").setTopicId(Uuid.fromString("GcaQDl2UTsCNs1p9s37XkQ"));
    metadataNodeManager.handleMessage(topicRecord);
    assertEquals("topicName", metadataNodeManager.getData().root().directory("topics", "topicName").file("name").contents());
    assertEquals("GcaQDl2UTsCNs1p9s37XkQ", metadataNodeManager.getData().root().directory("topics", "topicName").file("id").contents());
    assertEquals("topicName", metadataNodeManager.getData().root().directory("topicIds", "GcaQDl2UTsCNs1p9s37XkQ").file("name").contents());
    assertEquals("GcaQDl2UTsCNs1p9s37XkQ", metadataNodeManager.getData().root().directory("topicIds", "GcaQDl2UTsCNs1p9s37XkQ").file("id").contents());
    // Remove topic
    RemoveTopicRecord removeTopicRecord = new RemoveTopicRecord().setTopicId(Uuid.fromString("GcaQDl2UTsCNs1p9s37XkQ"));
    metadataNodeManager.handleMessage(removeTopicRecord);
    assertFalse(metadataNodeManager.getData().root().directory("topicIds").children().containsKey("GcaQDl2UTsCNs1p9s37XkQ"));
    assertFalse(metadataNodeManager.getData().root().directory("topics").children().containsKey("topicName"));
}
Also used : TopicRecord(org.apache.kafka.common.metadata.TopicRecord) RemoveTopicRecord(org.apache.kafka.common.metadata.RemoveTopicRecord) RemoveTopicRecord(org.apache.kafka.common.metadata.RemoveTopicRecord) Test(org.junit.jupiter.api.Test)

Example 3 with TopicRecord

use of org.apache.kafka.common.metadata.TopicRecord in project kafka by apache.

the class TopicsImageTest method testBasicLocalChanges.

@Test
public void testBasicLocalChanges() {
    int localId = 3;
    /* Changes already include in DELTA1_RECORDS and IMAGE1:
         * foo - topic id deleted
         * bar-0 - stay as follower with different partition epoch
         * baz-0 - new topic to leader
         */
    List<ApiMessageAndVersion> topicRecords = new ArrayList<>(DELTA1_RECORDS);
    // Create a new foo topic with a different id
    Uuid newFooId = Uuid.fromString("b66ybsWIQoygs01vdjH07A");
    topicRecords.add(new ApiMessageAndVersion(new TopicRecord().setName("foo").setTopicId(newFooId), TOPIC_RECORD.highestSupportedVersion()));
    topicRecords.add(newPartitionRecord(newFooId, 0, Arrays.asList(0, 1, 2)));
    topicRecords.add(newPartitionRecord(newFooId, 1, Arrays.asList(0, 1, localId)));
    // baz-1 - new partition to follower
    topicRecords.add(new ApiMessageAndVersion(new PartitionRecord().setPartitionId(1).setTopicId(BAZ_UUID).setReplicas(Arrays.asList(4, 2, localId)).setIsr(Arrays.asList(4, 2, localId)).setLeader(4).setLeaderEpoch(2).setPartitionEpoch(1), PARTITION_RECORD.highestSupportedVersion()));
    TopicsDelta delta = new TopicsDelta(IMAGE1);
    RecordTestUtils.replayAll(delta, topicRecords);
    LocalReplicaChanges changes = delta.localChanges(localId);
    assertEquals(new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("foo", 1))), changes.deletes());
    assertEquals(new HashSet<>(Arrays.asList(new TopicPartition("baz", 0))), changes.leaders().keySet());
    assertEquals(new HashSet<>(Arrays.asList(new TopicPartition("baz", 1), new TopicPartition("bar", 0), new TopicPartition("foo", 1))), changes.followers().keySet());
}
Also used : RemoveTopicRecord(org.apache.kafka.common.metadata.RemoveTopicRecord) TopicRecord(org.apache.kafka.common.metadata.TopicRecord) Uuid(org.apache.kafka.common.Uuid) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) TopicPartition(org.apache.kafka.common.TopicPartition) ArrayList(java.util.ArrayList) PartitionRecord(org.apache.kafka.common.metadata.PartitionRecord) Test(org.junit.jupiter.api.Test)

Example 4 with TopicRecord

use of org.apache.kafka.common.metadata.TopicRecord in project kafka by apache.

the class ReplicationControlManagerTest method testCreateTopics.

@Test
public void testCreateTopics() throws Exception {
    ReplicationControlTestContext ctx = new ReplicationControlTestContext();
    ReplicationControlManager replicationControl = ctx.replicationControl;
    CreateTopicsRequestData request = new CreateTopicsRequestData();
    request.topics().add(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1));
    ControllerResult<CreateTopicsResponseData> result = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse = new CreateTopicsResponseData();
    expectedResponse.topics().add(new CreatableTopicResult().setName("foo").setErrorCode(Errors.INVALID_REPLICATION_FACTOR.code()).setErrorMessage("Unable to replicate the partition 3 time(s): All " + "brokers are currently fenced."));
    assertEquals(expectedResponse, result.response());
    ctx.registerBrokers(0, 1, 2);
    ctx.unfenceBrokers(0, 1, 2);
    ControllerResult<CreateTopicsResponseData> result2 = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse2 = new CreateTopicsResponseData();
    expectedResponse2.topics().add(new CreatableTopicResult().setName("foo").setNumPartitions(1).setReplicationFactor((short) 3).setErrorMessage(null).setErrorCode((short) 0).setTopicId(result2.response().topics().find("foo").topicId()));
    assertEquals(expectedResponse2, result2.response());
    ctx.replay(result2.records());
    assertEquals(new PartitionRegistration(new int[] { 1, 2, 0 }, new int[] { 1, 2, 0 }, Replicas.NONE, Replicas.NONE, 1, 0, 0), replicationControl.getPartition(((TopicRecord) result2.records().get(0).message()).topicId(), 0));
    ControllerResult<CreateTopicsResponseData> result3 = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse3 = new CreateTopicsResponseData();
    expectedResponse3.topics().add(new CreatableTopicResult().setName("foo").setErrorCode(Errors.TOPIC_ALREADY_EXISTS.code()).setErrorMessage("Topic 'foo' already exists."));
    assertEquals(expectedResponse3, result3.response());
    Uuid fooId = result2.response().topics().find("foo").topicId();
    RecordTestUtils.assertBatchIteratorContains(asList(asList(new ApiMessageAndVersion(new PartitionRecord().setPartitionId(0).setTopicId(fooId).setReplicas(asList(1, 2, 0)).setIsr(asList(1, 2, 0)).setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()).setLeader(1).setLeaderEpoch(0).setPartitionEpoch(0), (short) 0), new ApiMessageAndVersion(new TopicRecord().setTopicId(fooId).setName("foo"), (short) 0))), ctx.replicationControl.iterator(Long.MAX_VALUE));
}
Also used : PartitionRegistration(org.apache.kafka.metadata.PartitionRegistration) TopicRecord(org.apache.kafka.common.metadata.TopicRecord) Uuid(org.apache.kafka.common.Uuid) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) PartitionRecord(org.apache.kafka.common.metadata.PartitionRecord) CreatableTopicResult(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 5 with TopicRecord

use of org.apache.kafka.common.metadata.TopicRecord in project kafka by apache.

the class MetadataRecordSerdeTest method testSerde.

@Test
public void testSerde() {
    TopicRecord topicRecord = new TopicRecord().setName("foo").setTopicId(Uuid.randomUuid());
    MetadataRecordSerde serde = new MetadataRecordSerde();
    for (short version = TopicRecord.LOWEST_SUPPORTED_VERSION; version <= TopicRecord.HIGHEST_SUPPORTED_VERSION; version++) {
        ApiMessageAndVersion messageAndVersion = new ApiMessageAndVersion(topicRecord, version);
        ObjectSerializationCache cache = new ObjectSerializationCache();
        int size = serde.recordSize(messageAndVersion, cache);
        ByteBuffer buffer = ByteBuffer.allocate(size);
        ByteBufferAccessor bufferAccessor = new ByteBufferAccessor(buffer);
        serde.write(messageAndVersion, cache, bufferAccessor);
        buffer.flip();
        assertEquals(size, buffer.remaining());
        ApiMessageAndVersion readMessageAndVersion = serde.read(bufferAccessor, size);
        assertEquals(messageAndVersion, readMessageAndVersion);
    }
}
Also used : ObjectSerializationCache(org.apache.kafka.common.protocol.ObjectSerializationCache) TopicRecord(org.apache.kafka.common.metadata.TopicRecord) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) ByteBufferAccessor(org.apache.kafka.common.protocol.ByteBufferAccessor) ByteBuffer(java.nio.ByteBuffer) Test(org.junit.jupiter.api.Test)

Aggregations

TopicRecord (org.apache.kafka.common.metadata.TopicRecord)6 ApiMessageAndVersion (org.apache.kafka.server.common.ApiMessageAndVersion)5 Test (org.junit.jupiter.api.Test)4 ArrayList (java.util.ArrayList)3 Uuid (org.apache.kafka.common.Uuid)3 RemoveTopicRecord (org.apache.kafka.common.metadata.RemoveTopicRecord)3 PartitionRegistration (org.apache.kafka.metadata.PartitionRegistration)3 CreatableTopicResult (org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult)2 PartitionRecord (org.apache.kafka.common.metadata.PartitionRecord)2 ByteBuffer (java.nio.ByteBuffer)1 HashMap (java.util.HashMap)1 List (java.util.List)1 OptionalInt (java.util.OptionalInt)1 TopicPartition (org.apache.kafka.common.TopicPartition)1 InvalidReplicationFactorException (org.apache.kafka.common.errors.InvalidReplicationFactorException)1 CreateTopicsRequestData (org.apache.kafka.common.message.CreateTopicsRequestData)1 CreatableReplicaAssignment (org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment)1 CreatableTopic (org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic)1 CreateTopicsResponseData (org.apache.kafka.common.message.CreateTopicsResponseData)1 ByteBufferAccessor (org.apache.kafka.common.protocol.ByteBufferAccessor)1