Examples with KafkaAdminClients io.strimzi.systemtest.kafkaclients.internalClients.KafkaAdminClients used on opensource projects

Example 1 with KafkaAdminClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaAdminClients in project strimzi-kafka-operator by strimzi.

the class ThrottlingQuotaST method testThrottlingQuotasCreateTopic.

/**
 * Test checks for throttling quotas set for user
 * on creation & deletion of topics and create partition operations.
 */
@ParallelTest
void testThrottlingQuotasCreateTopic(ExtensionContext extensionContext) {
    final String kafkaUsername = mapWithTestUsers.get(extensionContext.getDisplayName());
    final String createAdminName = "create-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String topicNamePrefix = classTopicPrefix + "-create";
    int topicsCountOverQuota = 500;
    setupKafkaUserInNamespace(extensionContext, kafkaUsername);
    KafkaAdminClients adminClientJob = new KafkaAdminClientsBuilder().withAdminName(createAdminName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(CLUSTER_NAME)).withTopicName(topicNamePrefix).withTopicCount(topicsCountOverQuota).withNamespaceName(namespace).withTopicOperation(AdminClientOperations.CREATE_TOPICS.toString()).withAdditionalConfig(KafkaAdminClients.getAdminClientScramConfig(namespace, kafkaUsername, 240000)).build();
    resourceManager.createResource(extensionContext, adminClientJob.defaultAdmin());
    String createPodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", createAdminName).get(0);
    PodUtils.waitUntilMessageIsInPodLogs(namespace, createPodName, "org.apache.kafka.common.errors.ThrottlingQuotaExceededException: The throttling quota has been exceeded.", GLOBAL_TIMEOUT);
    JobUtils.deleteJobWithWait(namespace, createAdminName);
    // Teardown delete created topics
    KafkaTopicUtils.deleteAllKafkaTopicsWithPrefix(namespace, topicNamePrefix);
}

Example 2 with KafkaAdminClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaAdminClients in project strimzi by strimzi.

the class ThrottlingQuotaST method testKafkaAdminTopicOperations.

@ParallelTest
void testKafkaAdminTopicOperations(ExtensionContext extensionContext) {
    final String kafkaUsername = mapWithTestUsers.get(extensionContext.getDisplayName());
    final String createAdminName = "create-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String deleteAdminName = "delete-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String listAdminName = "list-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String topicNamePrefix = classTopicPrefix + "-simple";
    int topicsCountBelowQuota = 100;
    setupKafkaUserInNamespace(extensionContext, kafkaUsername);
    // Create 'topicsCountBelowQuota' topics
    KafkaAdminClients adminClientJob = new KafkaAdminClientsBuilder().withAdminName(createAdminName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(CLUSTER_NAME)).withTopicName(topicNamePrefix).withTopicCount(topicsCountBelowQuota).withNamespaceName(namespace).withTopicOperation(AdminClientOperations.CREATE_TOPICS.toString()).withAdditionalConfig(KafkaAdminClients.getAdminClientScramConfig(namespace, kafkaUsername, 240000)).build();
    resourceManager.createResource(extensionContext, adminClientJob.defaultAdmin());
    ClientUtils.waitForClientSuccess(createAdminName, namespace, topicsCountBelowQuota);
    String createPodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", createAdminName).get(0);
    PodUtils.waitUntilMessageIsInPodLogs(namespace, createPodName, "All topics created");
    // List 'topicsCountBelowQuota' topics
    KafkaAdminClients adminClientListJob = new KafkaAdminClientsBuilder().withAdminName(listAdminName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(CLUSTER_NAME)).withNamespaceName(namespace).withTopicOperation(AdminClientOperations.LIST_TOPICS.toString()).withAdditionalConfig(KafkaAdminClients.getAdminClientScramConfig(namespace, kafkaUsername, 600000)).build();
    resourceManager.createResource(extensionContext, adminClientListJob.defaultAdmin());
    ClientUtils.waitForClientSuccess(listAdminName, namespace, 0);
    String listPodName = kubeClient().listPodNamesInSpecificNamespace(namespace, "job-name", listAdminName).get(0);
    PodUtils.waitUntilMessageIsInPodLogs(namespace, listPodName, topicNamePrefix + "-" + (topicsCountBelowQuota - 1));
    JobUtils.deleteJobWithWait(namespace, listAdminName);
    // Delete 'topicsCountBelowQuota' topics
    adminClientJob = new KafkaAdminClientsBuilder(adminClientJob).withAdminName(deleteAdminName).withTopicOperation(AdminClientOperations.DELETE_TOPICS.toString()).build();
    resourceManager.createResource(extensionContext, adminClientJob.defaultAdmin());
    ClientUtils.waitForClientSuccess(deleteAdminName, namespace, 10);
    String deletePodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", deleteAdminName).get(0);
    PodUtils.waitUntilMessageIsInPodLogs(namespace, deletePodName, "Successfully removed all " + topicsCountBelowQuota);
    // List topics after deletion
    resourceManager.createResource(extensionContext, adminClientListJob.defaultAdmin());
    ClientUtils.waitForClientSuccess(listAdminName, namespace, 0);
    listPodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", listAdminName).get(0);
    String afterDeletePodLogs = kubeClient(namespace).logsInSpecificNamespace(namespace, listPodName);
    assertThat(afterDeletePodLogs.contains(topicNamePrefix), is(false));
    assertThat(afterDeletePodLogs, not(containsString(topicNamePrefix)));
    JobUtils.deleteJobWithWait(namespace, createAdminName);
    JobUtils.deleteJobWithWait(namespace, listAdminName);
    JobUtils.deleteJobWithWait(namespace, deleteAdminName);
}

Example 3 with KafkaAdminClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaAdminClients in project strimzi by strimzi.

the class ThrottlingQuotaST method testThrottlingQuotasCreateTopic.

/**
 * Test checks for throttling quotas set for user
 * on creation & deletion of topics and create partition operations.
 */
@ParallelTest
void testThrottlingQuotasCreateTopic(ExtensionContext extensionContext) {
    final String kafkaUsername = mapWithTestUsers.get(extensionContext.getDisplayName());
    final String createAdminName = "create-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String topicNamePrefix = classTopicPrefix + "-create";
    int topicsCountOverQuota = 500;
    setupKafkaUserInNamespace(extensionContext, kafkaUsername);
    KafkaAdminClients adminClientJob = new KafkaAdminClientsBuilder().withAdminName(createAdminName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(CLUSTER_NAME)).withTopicName(topicNamePrefix).withTopicCount(topicsCountOverQuota).withNamespaceName(namespace).withTopicOperation(AdminClientOperations.CREATE_TOPICS.toString()).withAdditionalConfig(KafkaAdminClients.getAdminClientScramConfig(namespace, kafkaUsername, 240000)).build();
    resourceManager.createResource(extensionContext, adminClientJob.defaultAdmin());
    String createPodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", createAdminName).get(0);
    PodUtils.waitUntilMessageIsInPodLogs(namespace, createPodName, "org.apache.kafka.common.errors.ThrottlingQuotaExceededException: The throttling quota has been exceeded.", GLOBAL_TIMEOUT);
    JobUtils.deleteJobWithWait(namespace, createAdminName);
    // Teardown delete created topics
    KafkaTopicUtils.deleteAllKafkaTopicsWithPrefix(namespace, topicNamePrefix);
}

Example 4 with KafkaAdminClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaAdminClients in project strimzi by strimzi.

the class ThrottlingQuotaST method testThrottlingQuotasDeleteTopic.

@ParallelTest
void testThrottlingQuotasDeleteTopic(ExtensionContext extensionContext) {
    final String kafkaUsername = mapWithTestUsers.get(extensionContext.getDisplayName());
    final String createAdminName = "create-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String deleteAdminName = "delete-admin-" + mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String topicNamePrefix = classTopicPrefix + "-delete";
    int topicsCountOverQuota = 500;
    setupKafkaUserInNamespace(extensionContext, kafkaUsername);
    // Create many topics in multiple rounds using starting offset
    KafkaAdminClients createAdminClientJob = new KafkaAdminClientsBuilder().withAdminName(createAdminName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(CLUSTER_NAME)).withTopicName(topicNamePrefix).withTopicCount(100).withNamespaceName(namespace).withTopicOperation(AdminClientOperations.CREATE_TOPICS.toString()).withAdditionalConfig(KafkaAdminClients.getAdminClientScramConfig(namespace, kafkaUsername, 240000)).build();
    String createPodName;
    int offset = 100;
    int iterations = topicsCountOverQuota / 100;
    for (int i = 0; i < iterations; i++) {
        LOGGER.info("Executing {}/{} iteration.", i + 1, iterations);
        createAdminClientJob = new KafkaAdminClientsBuilder(createAdminClientJob).withTopicOffset(i * offset).build();
        resourceManager.createResource(extensionContext, createAdminClientJob.defaultAdmin());
        ClientUtils.waitForClientSuccess(createAdminName, namespace, 100);
        createPodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", createAdminName).get(0);
        PodUtils.waitUntilMessageIsInPodLogs(namespace, createPodName, "All topics created", Constants.GLOBAL_TIMEOUT);
        JobUtils.deleteJobWithWait(namespace, createAdminName);
    }
    // Test delete all topics at once - should fail on Throttling Quota limit
    KafkaAdminClients deleteAdminClientJob = new KafkaAdminClientsBuilder().withAdminName(deleteAdminName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(CLUSTER_NAME)).withTopicName(topicNamePrefix).withTopicCount(topicsCountOverQuota).withNamespaceName(namespace).withTopicOperation(AdminClientOperations.DELETE_TOPICS.toString()).withAdditionalConfig(KafkaAdminClients.getAdminClientScramConfig(namespace, kafkaUsername, 240000)).build();
    resourceManager.createResource(extensionContext, deleteAdminClientJob.defaultAdmin());
    String deletePodName = kubeClient(namespace).listPodNamesInSpecificNamespace(namespace, "job-name", deleteAdminName).get(0);
    PodUtils.waitUntilMessageIsInPodLogs(namespace, deletePodName, "org.apache.kafka.common.errors.ThrottlingQuotaExceededException: The throttling quota has been exceeded.", Constants.GLOBAL_TIMEOUT);
    JobUtils.deleteJobWithWait(namespace, deleteAdminName);
    // Teardown - delete all (remaining) topics (within Quota limits) in multiple rounds using starting offsets
    for (int i = 0; i < iterations; i++) {
        LOGGER.info("Executing {}/{} iteration for {}.", i + 1, iterations, deleteAdminName);
        deleteAdminClientJob = new KafkaAdminClientsBuilder(deleteAdminClientJob).withTopicOffset(i * offset).withTopicCount(100).withAdditionalConfig("").build();
        resourceManager.createResource(extensionContext, deleteAdminClientJob.defaultAdmin());
        ClientUtils.waitForClientSuccess(deleteAdminName, namespace, 10);
        JobUtils.deleteJobWithWait(namespace, deleteAdminName);
    }
}

Example 5 with KafkaAdminClients

use of io.strimzi.systemtest.kafkaclients.internalClients.KafkaAdminClients in project strimzi by strimzi.

the class ThrottlingQuotaST method testThrottlingQuotasDuringAllTopicOperations.

@ParallelTest
void testThrottlingQuotasDuringAllTopicOperations(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    final String createAdminName = "create-" + testStorage.getAdminName();
    final String alterAdminName = "alter-" + testStorage.getAdminName();
    final String deleteAdminName = "delete-" + testStorage.getAdminName();
    final String listAdminName = "list-" + testStorage.getAdminName();
    final String kafkaPodName = KafkaResources.kafkaPodName(sharedTestStorage.getClusterName(), 0);
    final String plainBootstrapName = KafkaResources.plainBootstrapAddress(sharedTestStorage.getClusterName());
    int numOfTopics = 25;
    int numOfPartitions = 100;
    int iterations = numOfTopics / 5;
    KafkaAdminClients createTopicJob = adminClientsBuilder.withAdminName(createAdminName).withTopicName(testStorage.getTopicName()).withTopicCount(numOfTopics).withPartitions(numOfPartitions).withAdminOperation(AdminClientOperation.CREATE_TOPICS).build();
    LOGGER.info("Creating {} topics with {} partitions, we should hit the quota", numOfTopics, numOfPartitions);
    resourceManager.createResource(extensionContext, createTopicJob.defaultAdmin());
    ClientUtils.waitForClientContainsMessage(createAdminName, testStorage.getNamespaceName(), THROTTLING_ERROR_MSG);
    KafkaTopicUtils.deleteAllKafkaTopicsByPrefixWithWait(testStorage.getNamespaceName(), testStorage.getTopicName());
    // we need to wait for all KafkaTopics to be deleted from Kafka before proceeding - using Kafka pod cli (with AdminClient props)
    KafkaTopicUtils.waitForTopicsByPrefixDeletionUsingPodCli(testStorage.getNamespaceName(), testStorage.getTopicName(), plainBootstrapName, kafkaPodName, createTopicJob.getAdditionalConfig());
    numOfPartitions = 5;
    createTopicJob = new KafkaAdminClientsBuilder(createTopicJob).withPartitions(numOfPartitions).build();
    LOGGER.info("Creating {} topics with {} partitions, the quota should not be exceeded", numOfTopics, numOfPartitions);
    resourceManager.createResource(extensionContext, createTopicJob.defaultAdmin());
    ClientUtils.waitForClientContainsMessage(createAdminName, testStorage.getNamespaceName(), "All topics created");
    KafkaAdminClients listTopicJob = new KafkaAdminClientsBuilder(createTopicJob).withAdminName(listAdminName).withTopicName("").withAdminOperation(AdminClientOperation.LIST_TOPICS).build();
    LOGGER.info("Listing topics after creation");
    resourceManager.createResource(extensionContext, listTopicJob.defaultAdmin());
    ClientUtils.waitForClientContainsMessage(listAdminName, testStorage.getNamespaceName(), testStorage.getTopicName() + "-" + (numOfTopics - 1));
    int partitionAlter = 25;
    KafkaAdminClients alterTopicsJob = new KafkaAdminClientsBuilder(createTopicJob).withAdminName(alterAdminName).withPartitions(partitionAlter).withAdminOperation(AdminClientOperation.UPDATE_TOPICS).build();
    LOGGER.info("Altering {} topics - setting partitions to {} - we should hit the quota", numOfTopics, partitionAlter);
    // because we are not hitting the quota, this should pass without a problem
    resourceManager.createResource(extensionContext, alterTopicsJob.defaultAdmin());
    ClientUtils.waitForClientContainsMessage(alterAdminName, testStorage.getNamespaceName(), THROTTLING_ERROR_MSG);
    // we need to set higher partitions - for case when we altered some topics before hitting the quota to 25 partitions
    partitionAlter = 30;
    int numOfTopicsIter = 5;
    alterTopicsJob = new KafkaAdminClientsBuilder(alterTopicsJob).withPartitions(partitionAlter).withTopicCount(numOfTopicsIter).build();
    for (int i = 0; i < iterations; i++) {
        alterTopicsJob = new KafkaAdminClientsBuilder(alterTopicsJob).withTopicCount(numOfTopicsIter).withTopicOffset(numOfTopicsIter * i).build();
        LOGGER.info("Altering {} topics with offset {} - setting partitions to {} - we should not hit the quota", numOfTopicsIter, numOfTopicsIter * i, partitionAlter);
        resourceManager.createResource(extensionContext, alterTopicsJob.defaultAdmin());
        ClientUtils.waitForClientContainsMessage(alterAdminName, testStorage.getNamespaceName(), "All topics altered");
    }
    // delete few topics
    KafkaAdminClients deleteTopicsJob = adminClientsBuilder.withTopicName(testStorage.getTopicName()).withAdminName(deleteAdminName).withAdminOperation(AdminClientOperation.DELETE_TOPICS).withTopicCount(numOfTopicsIter).build();
    LOGGER.info("Deleting first {} topics, we will not hit the quota", numOfTopicsIter);
    resourceManager.createResource(extensionContext, deleteTopicsJob.defaultAdmin());
    ClientUtils.waitForClientContainsMessage(deleteAdminName, testStorage.getNamespaceName(), "Successfully removed all " + numOfTopicsIter);
    int remainingTopics = numOfTopics - numOfTopicsIter;
    deleteTopicsJob = new KafkaAdminClientsBuilder(deleteTopicsJob).withTopicCount(remainingTopics).withTopicOffset(numOfTopicsIter).build();
    LOGGER.info("Trying to remove all remaining {} topics with offset of {} - we should hit the quota", remainingTopics, numOfTopicsIter);
    resourceManager.createResource(extensionContext, deleteTopicsJob.defaultAdmin());
    ClientUtils.waitForClientContainsMessage(deleteAdminName, testStorage.getNamespaceName(), THROTTLING_ERROR_MSG);
    LOGGER.info("Because we hit quota, removing the remaining topics through console");
    KafkaTopicUtils.deleteAllKafkaTopicsByPrefixWithWait(testStorage.getNamespaceName(), testStorage.getTopicName());
    // we need to wait for all KafkaTopics to be deleted from Kafka before proceeding - using Kafka pod cli (with AdminClient props)
    KafkaTopicUtils.waitForTopicsByPrefixDeletionUsingPodCli(testStorage.getNamespaceName(), testStorage.getTopicName(), plainBootstrapName, kafkaPodName, createTopicJob.getAdditionalConfig());
    // List topics after deletion
    resourceManager.createResource(extensionContext, listTopicJob.defaultAdmin());
    ClientUtils.waitForClientSuccess(listAdminName, testStorage.getNamespaceName(), 0, false);
    String listPodName = PodUtils.getPodNameByPrefix(testStorage.getNamespaceName(), listAdminName);
    String afterDeletePodLogs = kubeClient().logsInSpecificNamespace(testStorage.getNamespaceName(), listPodName);
    assertFalse(afterDeletePodLogs.contains(testStorage.getTopicName()));
    JobUtils.deleteJobWithWait(testStorage.getNamespaceName(), listAdminName);
}