Search in sources :

Example 1 with CreatableReplicaAssignmentCollection

use of org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection in project kafka by apache.

the class QuorumControllerTest method testMissingInMemorySnapshot.

@Test
public void testMissingInMemorySnapshot() throws Exception {
    int numBrokers = 3;
    int numPartitions = 3;
    String topicName = "topic-name";
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty());
        QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
        QuorumController controller = controlEnv.activeController();
        Map<Integer, Long> brokerEpochs = registerBrokers(controller, numBrokers);
        // Create a lot of partitions
        List<CreatableReplicaAssignment> partitions = IntStream.range(0, numPartitions).mapToObj(partitionIndex -> new CreatableReplicaAssignment().setPartitionIndex(partitionIndex).setBrokerIds(Arrays.asList(0, 1, 2))).collect(Collectors.toList());
        Uuid topicId = controller.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName(topicName).setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(partitions.iterator()))).iterator()))).get().topics().find(topicName).topicId();
        // Create a lot of alter isr
        List<AlterIsrRequestData.PartitionData> alterIsrs = IntStream.range(0, numPartitions).mapToObj(partitionIndex -> {
            PartitionRegistration partitionRegistration = controller.replicationControl().getPartition(topicId, partitionIndex);
            return new AlterIsrRequestData.PartitionData().setPartitionIndex(partitionIndex).setLeaderEpoch(partitionRegistration.leaderEpoch).setCurrentIsrVersion(partitionRegistration.partitionEpoch).setNewIsr(Arrays.asList(0, 1));
        }).collect(Collectors.toList());
        AlterIsrRequestData.TopicData topicData = new AlterIsrRequestData.TopicData().setName(topicName);
        topicData.partitions().addAll(alterIsrs);
        int leaderId = 0;
        AlterIsrRequestData alterIsrRequest = new AlterIsrRequestData().setBrokerId(leaderId).setBrokerEpoch(brokerEpochs.get(leaderId));
        alterIsrRequest.topics().add(topicData);
        logEnv.logManagers().get(0).resignAfterNonAtomicCommit();
        int oldClaimEpoch = controller.curClaimEpoch();
        assertThrows(ExecutionException.class, () -> controller.alterIsr(alterIsrRequest).get());
        // Wait for the controller to become active again
        assertSame(controller, controlEnv.activeController());
        assertTrue(oldClaimEpoch < controller.curClaimEpoch(), String.format("oldClaimEpoch = %s, newClaimEpoch = %s", oldClaimEpoch, controller.curClaimEpoch()));
        // Since the alterIsr partially failed we expect to see
        // some partitions to still have 2 in the ISR.
        int partitionsWithReplica2 = Utils.toList(controller.replicationControl().brokersToIsrs().partitionsWithBrokerInIsr(2)).size();
        int partitionsWithReplica0 = Utils.toList(controller.replicationControl().brokersToIsrs().partitionsWithBrokerInIsr(0)).size();
        assertEquals(numPartitions, partitionsWithReplica0);
        assertNotEquals(0, partitionsWithReplica2);
        assertTrue(partitionsWithReplica0 > partitionsWithReplica2, String.format("partitionsWithReplica0 = %s, partitionsWithReplica2 = %s", partitionsWithReplica0, partitionsWithReplica2));
    }
}
Also used : Arrays(java.util.Arrays) BrokerRegistrationRequestData(org.apache.kafka.common.message.BrokerRegistrationRequestData) BrokerIdNotRegisteredException(org.apache.kafka.common.errors.BrokerIdNotRegisteredException) ConfigurationControlManagerTest.entry(org.apache.kafka.controller.ConfigurationControlManagerTest.entry) Spliterators(java.util.Spliterators) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) ElectLeadersRequestData(org.apache.kafka.common.message.ElectLeadersRequestData) Listener(org.apache.kafka.common.message.BrokerRegistrationRequestData.Listener) CreatePartitionsTopicResult(org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult) Future(java.util.concurrent.Future) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) CreatePartitionsTopic(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic) BufferSupplier(org.apache.kafka.common.utils.BufferSupplier) Map(java.util.Map) SET(org.apache.kafka.clients.admin.AlterConfigOp.OpType.SET) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) ListPartitionReassignmentsRequestData(org.apache.kafka.common.message.ListPartitionReassignmentsRequestData) TopicRecord(org.apache.kafka.common.metadata.TopicRecord) RawSnapshotReader(org.apache.kafka.snapshot.RawSnapshotReader) TestUtils(org.apache.kafka.test.TestUtils) PartitionRegistration(org.apache.kafka.metadata.PartitionRegistration) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) CountDownLatch(java.util.concurrent.CountDownLatch) ConfigResourceExistenceChecker(org.apache.kafka.controller.QuorumController.ConfigResourceExistenceChecker) List(java.util.List) TopicIdPartition(org.apache.kafka.controller.BrokersToIsrs.TopicIdPartition) AllocateProducerIdsRequestData(org.apache.kafka.common.message.AllocateProducerIdsRequestData) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) BrokerEndpoint(org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint) Optional(java.util.Optional) Errors(org.apache.kafka.common.protocol.Errors) UnknownTopicOrPartitionException(org.apache.kafka.common.errors.UnknownTopicOrPartitionException) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) Spliterator(java.util.Spliterator) IntStream(java.util.stream.IntStream) Uuid(org.apache.kafka.common.Uuid) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) CONFIGS(org.apache.kafka.controller.ConfigurationControlManagerTest.CONFIGS) ListPartitionReassignmentsResponseData(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData) LocalLogManagerTestEnv(org.apache.kafka.metalog.LocalLogManagerTestEnv) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) ApiError(org.apache.kafka.common.requests.ApiError) TOPIC(org.apache.kafka.common.config.ConfigResource.Type.TOPIC) ConfigResource(org.apache.kafka.common.config.ConfigResource) BrokerHeartbeatReply(org.apache.kafka.metadata.BrokerHeartbeatReply) AlterPartitionReassignmentsRequestData(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) StreamSupport(java.util.stream.StreamSupport) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) SnapshotReader(org.apache.kafka.snapshot.SnapshotReader) Utils(org.apache.kafka.common.utils.Utils) RecordsSnapshotReader(org.apache.kafka.snapshot.RecordsSnapshotReader) MetadataRecordSerde(org.apache.kafka.metadata.MetadataRecordSerde) BROKER0(org.apache.kafka.controller.ConfigurationControlManagerTest.BROKER0) TimeoutException(org.apache.kafka.common.errors.TimeoutException) CreatableReplicaAssignment(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment) Iterator(java.util.Iterator) BrokerRegistrationReply(org.apache.kafka.metadata.BrokerRegistrationReply) ElectLeadersResponseData(org.apache.kafka.common.message.ElectLeadersResponseData) BrokerHeartbeatRequestData(org.apache.kafka.common.message.BrokerHeartbeatRequestData) BROKER(org.apache.kafka.common.config.ConfigResource.Type.BROKER) Batch(org.apache.kafka.raft.Batch) Assertions.assertSame(org.junit.jupiter.api.Assertions.assertSame) ExecutionException(java.util.concurrent.ExecutionException) AlterPartitionReassignmentsResponseData(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) RecordTestUtils(org.apache.kafka.metadata.RecordTestUtils) ReassignableTopic(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignableTopic) PartitionRecord(org.apache.kafka.common.metadata.PartitionRecord) AlterIsrRequestData(org.apache.kafka.common.message.AlterIsrRequestData) RegisterBrokerRecord(org.apache.kafka.common.metadata.RegisterBrokerRecord) HOURS(java.util.concurrent.TimeUnit.HOURS) Collections(java.util.Collections) Timeout(org.junit.jupiter.api.Timeout) ListenerCollection(org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection) BrokerEndpointCollection(org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpointCollection) ProducerIdsRecord(org.apache.kafka.common.metadata.ProducerIdsRecord) PartitionRegistration(org.apache.kafka.metadata.PartitionRegistration) LocalLogManagerTestEnv(org.apache.kafka.metalog.LocalLogManagerTestEnv) AlterIsrRequestData(org.apache.kafka.common.message.AlterIsrRequestData) BrokerEndpoint(org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) Uuid(org.apache.kafka.common.Uuid) CreatableReplicaAssignment(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) Test(org.junit.jupiter.api.Test)

Example 2 with CreatableReplicaAssignmentCollection

use of org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection in project kafka by apache.

the class InternalTopicManagerTest method shouldThrowInformativeExceptionForOlderBrokers.

@Test
public void shouldThrowInformativeExceptionForOlderBrokers() {
    final AdminClient admin = new MockAdminClient() {

        @Override
        public CreateTopicsResult createTopics(final Collection<NewTopic> newTopics, final CreateTopicsOptions options) {
            final CreatableTopic topicToBeCreated = new CreatableTopic();
            topicToBeCreated.setAssignments(new CreatableReplicaAssignmentCollection());
            topicToBeCreated.setNumPartitions((short) 1);
            // set unsupported replication factor for older brokers
            topicToBeCreated.setReplicationFactor((short) -1);
            final CreatableTopicCollection topicsToBeCreated = new CreatableTopicCollection();
            topicsToBeCreated.add(topicToBeCreated);
            try {
                new CreateTopicsRequest.Builder(new CreateTopicsRequestData().setTopics(topicsToBeCreated).setTimeoutMs(0).setValidateOnly(options.shouldValidateOnly())).build(// pass in old unsupported request version for old brokers
                (short) 3);
                throw new IllegalStateException("Building CreateTopicRequest should have thrown.");
            } catch (final UnsupportedVersionException expected) {
                final KafkaFutureImpl<TopicMetadataAndConfig> future = new KafkaFutureImpl<>();
                future.completeExceptionally(expected);
                return new CreateTopicsResult(Collections.singletonMap(topic1, future)) {
                };
            }
        }
    };
    final StreamsConfig streamsConfig = new StreamsConfig(config);
    final InternalTopicManager topicManager = new InternalTopicManager(time, admin, streamsConfig);
    final InternalTopicConfig topicConfig = new RepartitionTopicConfig(topic1, Collections.emptyMap());
    topicConfig.setNumberOfPartitions(1);
    final StreamsException exception = assertThrows(StreamsException.class, () -> topicManager.makeReady(Collections.singletonMap(topic1, topicConfig)));
    assertThat(exception.getMessage(), equalTo("Could not create topic " + topic1 + ", because brokers don't support configuration replication.factor=-1." + " You can change the replication.factor config or upgrade your brokers to version 2.4 or newer to avoid this error."));
}
Also used : StreamsException(org.apache.kafka.streams.errors.StreamsException) MockAdminClient(org.apache.kafka.clients.admin.MockAdminClient) CreateTopicsOptions(org.apache.kafka.clients.admin.CreateTopicsOptions) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) CreateTopicsRequest(org.apache.kafka.common.requests.CreateTopicsRequest) CreateTopicsResult(org.apache.kafka.clients.admin.CreateTopicsResult) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) Collection(java.util.Collection) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) AdminClient(org.apache.kafka.clients.admin.AdminClient) MockAdminClient(org.apache.kafka.clients.admin.MockAdminClient) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException) StreamsConfig(org.apache.kafka.streams.StreamsConfig) Test(org.junit.Test)

Example 3 with CreatableReplicaAssignmentCollection

use of org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection in project kafka by apache.

the class QuorumControllerTest method testSnapshotConfiguration.

@Test
public void testSnapshotConfiguration() throws Throwable {
    final int numBrokers = 4;
    final int maxNewRecordBytes = 4;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    Uuid fooId;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, builder -> {
            builder.setConfigDefs(CONFIGS).setSnapshotMaxNewRecordBytes(maxNewRecordBytes);
        })) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
            }
            for (int i = 0; i < numBrokers - 1; i++) {
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            fooId = fooData.topics().find("foo").topicId();
            active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForLatestSnapshot());
            checkSnapshotSubcontent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
}
Also used : BrokerHeartbeatReply(org.apache.kafka.metadata.BrokerHeartbeatReply) ListenerCollection(org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection) LocalLogManagerTestEnv(org.apache.kafka.metalog.LocalLogManagerTestEnv) Listener(org.apache.kafka.common.message.BrokerRegistrationRequestData.Listener) HashMap(java.util.HashMap) BrokerRegistrationRequestData(org.apache.kafka.common.message.BrokerRegistrationRequestData) BrokerRegistrationReply(org.apache.kafka.metadata.BrokerRegistrationReply) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) BrokerEndpoint(org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) Uuid(org.apache.kafka.common.Uuid) BrokerHeartbeatRequestData(org.apache.kafka.common.message.BrokerHeartbeatRequestData) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) CreatableReplicaAssignment(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) AllocateProducerIdsRequestData(org.apache.kafka.common.message.AllocateProducerIdsRequestData) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) Test(org.junit.jupiter.api.Test)

Example 4 with CreatableReplicaAssignmentCollection

use of org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection in project kafka by apache.

the class QuorumControllerTest method testSnapshotSaveAndLoad.

@Test
public void testSnapshotSaveAndLoad() throws Throwable {
    final int numBrokers = 4;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    RawSnapshotReader reader = null;
    Uuid fooId;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
            }
            for (int i = 0; i < numBrokers - 1; i++) {
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            fooId = fooData.topics().find("foo").topicId();
            active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            reader = logEnv.waitForSnapshot(snapshotLogOffset);
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(reader);
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.of(reader))) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForSnapshot(snapshotLogOffset));
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
}
Also used : Listener(org.apache.kafka.common.message.BrokerRegistrationRequestData.Listener) HashMap(java.util.HashMap) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) ApiMessageAndVersion(org.apache.kafka.server.common.ApiMessageAndVersion) AllocateProducerIdsRequestData(org.apache.kafka.common.message.AllocateProducerIdsRequestData) BrokerHeartbeatReply(org.apache.kafka.metadata.BrokerHeartbeatReply) ListenerCollection(org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection) LocalLogManagerTestEnv(org.apache.kafka.metalog.LocalLogManagerTestEnv) RawSnapshotReader(org.apache.kafka.snapshot.RawSnapshotReader) BrokerRegistrationRequestData(org.apache.kafka.common.message.BrokerRegistrationRequestData) BrokerRegistrationReply(org.apache.kafka.metadata.BrokerRegistrationReply) BrokerEndpoint(org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint) Uuid(org.apache.kafka.common.Uuid) BrokerHeartbeatRequestData(org.apache.kafka.common.message.BrokerHeartbeatRequestData) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) CreatableReplicaAssignment(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) Test(org.junit.jupiter.api.Test)

Example 5 with CreatableReplicaAssignmentCollection

use of org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection in project kafka by apache.

the class QuorumControllerTest method testSnapshotOnlyAfterConfiguredMinBytes.

@Test
public void testSnapshotOnlyAfterConfiguredMinBytes() throws Throwable {
    final int numBrokers = 4;
    final int maxNewRecordBytes = 1000;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, builder -> builder.setConfigDefs(CONFIGS).setSnapshotMaxNewRecordBytes(maxNewRecordBytes))) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            assertTrue(logEnv.appendedBytes() < maxNewRecordBytes, String.format("%s appended bytes is not less than %s max new record bytes", logEnv.appendedBytes(), maxNewRecordBytes));
            // Keep creating topic until we reached the max bytes limit
            int counter = 0;
            while (logEnv.appendedBytes() < maxNewRecordBytes) {
                counter += 1;
                String topicName = String.format("foo-%s", counter);
                active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName(topicName).setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            }
            logEnv.waitForLatestSnapshot();
        }
    }
}
Also used : BrokerHeartbeatReply(org.apache.kafka.metadata.BrokerHeartbeatReply) ListenerCollection(org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection) LocalLogManagerTestEnv(org.apache.kafka.metalog.LocalLogManagerTestEnv) Listener(org.apache.kafka.common.message.BrokerRegistrationRequestData.Listener) HashMap(java.util.HashMap) BrokerRegistrationRequestData(org.apache.kafka.common.message.BrokerRegistrationRequestData) BrokerRegistrationReply(org.apache.kafka.metadata.BrokerRegistrationReply) BrokerEndpoint(org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) BrokerHeartbeatRequestData(org.apache.kafka.common.message.BrokerHeartbeatRequestData) CreatableTopic(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic) CreatableReplicaAssignment(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) CreatableReplicaAssignmentCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection) Test(org.junit.jupiter.api.Test)

Aggregations

CreateTopicsRequestData (org.apache.kafka.common.message.CreateTopicsRequestData)5 CreatableReplicaAssignmentCollection (org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignmentCollection)5 CreatableTopic (org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic)5 HashMap (java.util.HashMap)4 BrokerHeartbeatRequestData (org.apache.kafka.common.message.BrokerHeartbeatRequestData)4 BrokerRegistrationRequestData (org.apache.kafka.common.message.BrokerRegistrationRequestData)4 Listener (org.apache.kafka.common.message.BrokerRegistrationRequestData.Listener)4 ListenerCollection (org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection)4 CreatableReplicaAssignment (org.apache.kafka.common.message.CreateTopicsRequestData.CreatableReplicaAssignment)4 CreatableTopicCollection (org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection)4 Uuid (org.apache.kafka.common.Uuid)3 AllocateProducerIdsRequestData (org.apache.kafka.common.message.AllocateProducerIdsRequestData)3 BrokerEndpoint (org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint)3 BrokerHeartbeatReply (org.apache.kafka.metadata.BrokerHeartbeatReply)3 BrokerRegistrationReply (org.apache.kafka.metadata.BrokerRegistrationReply)3 LocalLogManagerTestEnv (org.apache.kafka.metalog.LocalLogManagerTestEnv)3 Test (org.junit.jupiter.api.Test)3 CreateTopicsResponseData (org.apache.kafka.common.message.CreateTopicsResponseData)2 ApiMessageAndVersion (org.apache.kafka.server.common.ApiMessageAndVersion)2 Arrays (java.util.Arrays)1