Search in sources :

Example 6 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method onRebalancing.

/**
 * This method handles the transition from {@code Rebalancing} state.
 * It starts a periodic timer in order to check the status of the ongoing rebalance processing on Cruise Control side.
 * In order to do that, it calls the related Cruise Control REST API about asking the user task status.
 * When the rebalance is finished, the next state is {@code Ready}.
 * If the user sets the strimzi.io/rebalance annotation to 'stop', it calls the Cruise Control REST API for stopping the ongoing task
 * and then transitions to the {@code Stopped} state.
 * If the user sets any other values for the strimzi.io/rebalance annotation, it is just ignored and the user task checks continue.
 * This method holds the lock until the rebalance is finished, the ongoing task is stopped or any exception is raised.
 *
 * @param reconciliation Reconciliation information
 * @param host Cruise Control service to which sending the REST API requests
 * @param apiClient Cruise Control REST API client instance
 * @param kafkaRebalance Current {@code KafkaRebalance} resource
 * @param rebalanceAnnotation The current value for the strimzi.io/rebalance annotation
 * @return a Future with the next {@code MapAndStatus<ConfigMap, KafkaRebalanceStatus>} including the state
 */
private Future<MapAndStatus<ConfigMap, KafkaRebalanceStatus>> onRebalancing(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, KafkaRebalanceAnnotation rebalanceAnnotation) {
    Promise<MapAndStatus<ConfigMap, KafkaRebalanceStatus>> p = Promise.promise();
    if (rebalanceAnnotation == KafkaRebalanceAnnotation.none) {
        LOGGER.infoCr(reconciliation, "Starting Cruise Control rebalance user task status timer");
        String sessionId = kafkaRebalance.getStatus().getSessionId();
        AtomicInteger ccApiErrorCount = new AtomicInteger();
        vertx.setPeriodic(REBALANCE_POLLING_TIMER_MS, t -> {
            // Check that we have not already failed to contact the API beyond the allowed number of times.
            if (ccApiErrorCount.get() >= MAX_API_RETRIES) {
                vertx.cancelTimer(t);
                p.fail(new CruiseControlRestException("Unable to reach Cruise Control API after " + MAX_API_RETRIES + " attempts"));
            }
            kafkaRebalanceOperator.getAsync(kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName()).onSuccess(currentKafkaRebalance -> {
                // Checking that the resource was not deleted between periodic polls
                if (currentKafkaRebalance != null) {
                    // Safety check as timer might be called again (from a delayed timer firing)
                    if (state(currentKafkaRebalance) == KafkaRebalanceState.Rebalancing) {
                        if (rebalanceAnnotation(reconciliation, currentKafkaRebalance) == KafkaRebalanceAnnotation.stop) {
                            LOGGER.debugCr(reconciliation, "Stopping current Cruise Control rebalance user task");
                            vertx.cancelTimer(t);
                            apiClient.stopExecution(host, CruiseControl.REST_API_PORT).onSuccess(r -> p.complete(buildRebalanceStatus(null, KafkaRebalanceState.Stopped, validate(reconciliation, kafkaRebalance)))).onFailure(e -> {
                                LOGGER.errorCr(reconciliation, "Cruise Control stopping execution failed", e.getCause());
                                p.fail(e.getCause());
                            });
                        } else {
                            LOGGER.infoCr(reconciliation, "Getting Cruise Control rebalance user task status");
                            Set<Condition> conditions = validate(reconciliation, kafkaRebalance);
                            validateAnnotation(reconciliation, conditions, KafkaRebalanceState.Rebalancing, rebalanceAnnotation(reconciliation, currentKafkaRebalance), kafkaRebalance);
                            apiClient.getUserTaskStatus(host, CruiseControl.REST_API_PORT, sessionId).onSuccess(cruiseControlResponse -> {
                                JsonObject taskStatusJson = cruiseControlResponse.getJson();
                                CruiseControlUserTaskStatus taskStatus = CruiseControlUserTaskStatus.lookup(taskStatusJson.getString("Status"));
                                switch(taskStatus) {
                                    case COMPLETED:
                                        vertx.cancelTimer(t);
                                        LOGGER.infoCr(reconciliation, "Rebalance ({}) is now complete", sessionId);
                                        p.complete(buildRebalanceStatus(kafkaRebalance, null, KafkaRebalanceState.Ready, taskStatusJson, conditions));
                                        break;
                                    case COMPLETED_WITH_ERROR:
                                        // TODO: There doesn't seem to be a way to retrieve the actual error message from the user tasks endpoint?
                                        // We may need to propose an upstream PR for this.
                                        // TODO: Once we can get the error details we need to add an error field to the Rebalance Status to hold
                                        // details of any issues while rebalancing.
                                        LOGGER.errorCr(reconciliation, "Rebalance ({}) optimization proposal has failed to complete", sessionId);
                                        vertx.cancelTimer(t);
                                        p.complete(buildRebalanceStatus(sessionId, KafkaRebalanceState.NotReady, conditions));
                                        break;
                                    case // Rebalance is still in progress
                                    IN_EXECUTION:
                                        // the proposal is complete but the optimisation proposal summary will be missing.
                                        if (currentKafkaRebalance.getStatus().getOptimizationResult() == null || currentKafkaRebalance.getStatus().getOptimizationResult().isEmpty()) {
                                            LOGGER.infoCr(reconciliation, "Rebalance ({}) optimization proposal is now ready and has been added to the status", sessionId);
                                            // Cancel the timer so that the status is returned and updated.
                                            vertx.cancelTimer(t);
                                            p.complete(buildRebalanceStatus(kafkaRebalance, sessionId, KafkaRebalanceState.Rebalancing, taskStatusJson, conditions));
                                        }
                                        ccApiErrorCount.set(0);
                                        // We can then update the status at this point.
                                        break;
                                    case // Rebalance proposal is still being calculated
                                    ACTIVE:
                                        // If a rebalance(dryrun=false) was called and the proposal is still being prepared then the task
                                        // will be in an ACTIVE state. When the proposal is ready it will shift to IN_EXECUTION and we will
                                        // check that the optimisation proposal is added to the status on the next reconcile.
                                        LOGGER.infoCr(reconciliation, "Rebalance ({}) optimization proposal is still being prepared", sessionId);
                                        ccApiErrorCount.set(0);
                                        break;
                                    default:
                                        LOGGER.errorCr(reconciliation, "Unexpected state {}", taskStatus);
                                        vertx.cancelTimer(t);
                                        p.fail("Unexpected state " + taskStatus);
                                        break;
                                }
                            }).onFailure(e -> {
                                LOGGER.errorCr(reconciliation, "Cruise Control getting rebalance task status failed", e.getCause());
                                // To make sure this error is not just a temporary problem with the network we retry several times.
                                // If the number of errors pass the MAX_API_ERRORS limit then the period method will fail the promise.
                                ccApiErrorCount.getAndIncrement();
                            });
                        }
                    } else {
                        p.complete(new MapAndStatus<>(null, currentKafkaRebalance.getStatus()));
                    }
                } else {
                    LOGGER.debugCr(reconciliation, "Rebalance resource was deleted, stopping the request time");
                    vertx.cancelTimer(t);
                    p.complete();
                }
            }).onFailure(e -> {
                LOGGER.errorCr(reconciliation, "Cruise Control getting rebalance resource failed", e.getCause());
                vertx.cancelTimer(t);
                p.fail(e.getCause());
            });
        });
    } else {
        p.complete(new MapAndStatus<>(null, kafkaRebalance.getStatus()));
    }
    return p.future();
}
Also used : Arrays(java.util.Arrays) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) ANNO_STRIMZI_IO_REBALANCE(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_REBALANCE) Watcher(io.fabric8.kubernetes.client.Watcher) Annotations(io.strimzi.operator.common.Annotations) CruiseControlConfiguration(io.strimzi.operator.cluster.model.CruiseControlConfiguration) AbstractRebalanceOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.AbstractRebalanceOptions) Resource(io.fabric8.kubernetes.client.dsl.Resource) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CruiseControlRebalanceResponse(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRebalanceResponse) Map(java.util.Map) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) JsonObject(io.vertx.core.json.JsonObject) CruiseControlRebalanceKeys(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRebalanceKeys) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) KubernetesClientException(io.fabric8.kubernetes.client.KubernetesClientException) AbstractOperator(io.strimzi.operator.common.AbstractOperator) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) KafkaRebalance(io.strimzi.api.kafka.model.KafkaRebalance) SecretOperator(io.strimzi.operator.common.operator.resource.SecretOperator) Set(java.util.Set) KafkaRebalanceList(io.strimzi.api.kafka.KafkaRebalanceList) KafkaRebalanceAnnotation(io.strimzi.api.kafka.model.balancing.KafkaRebalanceAnnotation) Future(io.vertx.core.Future) Collectors(java.util.stream.Collectors) NoSuchResourceException(io.strimzi.operator.cluster.model.NoSuchResourceException) CruiseControlUserTaskStatus(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlUserTaskStatus) KafkaRebalanceStatusBuilder(io.strimzi.api.kafka.model.status.KafkaRebalanceStatusBuilder) CruiseControlResources(io.strimzi.api.kafka.model.CruiseControlResources) AddBrokerOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.AddBrokerOptions) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Stream(java.util.stream.Stream) CruiseControlApi(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApi) Secret(io.fabric8.kubernetes.api.model.Secret) Optional(java.util.Optional) Condition(io.strimzi.api.kafka.model.status.Condition) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) KafkaList(io.strimzi.api.kafka.KafkaList) KafkaRebalanceState(io.strimzi.api.kafka.model.balancing.KafkaRebalanceState) KafkaRebalanceBuilder(io.strimzi.api.kafka.model.KafkaRebalanceBuilder) HashMap(java.util.HashMap) CruiseControlApiImpl(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApiImpl) KafkaRebalanceSpec(io.strimzi.api.kafka.model.KafkaRebalanceSpec) ArrayList(java.util.ArrayList) KafkaRebalanceMode(io.strimzi.api.kafka.model.balancing.KafkaRebalanceMode) HashSet(java.util.HashSet) WatcherException(io.fabric8.kubernetes.client.WatcherException) CompositeFuture(io.vertx.core.CompositeFuture) ConfigMapOperator(io.strimzi.operator.common.operator.resource.ConfigMapOperator) CrdOperator(io.strimzi.operator.common.operator.resource.CrdOperator) RebalanceOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.RebalanceOptions) CruiseControl(io.strimzi.operator.cluster.model.CruiseControl) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) CruiseControlLoadParameters(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlLoadParameters) ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap) KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder) Reconciliation(io.strimzi.operator.common.Reconciliation) JsonArray(io.vertx.core.json.JsonArray) HTTP_DEFAULT_IDLE_TIMEOUT_SECONDS(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApiImpl.HTTP_DEFAULT_IDLE_TIMEOUT_SECONDS) StatusDiff(io.strimzi.operator.cluster.model.StatusDiff) Util(io.strimzi.operator.common.Util) RemoveBrokerOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.RemoveBrokerOptions) AbstractWatchableStatusedResourceOperator(io.strimzi.operator.common.operator.resource.AbstractWatchableStatusedResourceOperator) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Kafka(io.strimzi.api.kafka.model.Kafka) CruiseControlRestException(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRestException) Collections(java.util.Collections) Set(java.util.Set) HashSet(java.util.HashSet) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CruiseControlUserTaskStatus(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlUserTaskStatus) JsonObject(io.vertx.core.json.JsonObject) CruiseControlRestException(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRestException)

Example 7 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperatorTest method krNewToPendingProposalToProposalReady.

private void krNewToPendingProposalToProposalReady(VertxTestContext context, int pendingCalls, CruiseControlEndpoints endpoint, KafkaRebalance kr) throws IOException, URISyntaxException {
    // Setup the rebalance endpoint with the number of pending calls before a response is received.
    MockCruiseControl.setupCCRebalanceResponse(ccServer, pendingCalls, endpoint);
    Crds.kafkaRebalanceOperation(client).inNamespace(CLUSTER_NAMESPACE).create(kr);
    when(mockKafkaOps.getAsync(CLUSTER_NAMESPACE, CLUSTER_NAME)).thenReturn(Future.succeededFuture(kafka));
    mockSecretResources();
    mockRebalanceOperator(mockRebalanceOps, mockCmOps, CLUSTER_NAMESPACE, kr.getMetadata().getName(), client);
    Checkpoint checkpoint = context.checkpoint();
    kcrao.reconcileRebalance(new Reconciliation("test-trigger", KafkaRebalance.RESOURCE_KIND, CLUSTER_NAMESPACE, kr.getMetadata().getName()), kr).onComplete(context.succeeding(v -> {
        // the resource moved from New to PendingProposal (due to the configured Mock server pending calls)
        assertState(context, client, CLUSTER_NAMESPACE, kr.getMetadata().getName(), KafkaRebalanceState.PendingProposal);
    })).compose(v -> {
        // trigger another reconcile to process the PendingProposal state
        KafkaRebalance kr1 = Crds.kafkaRebalanceOperation(client).inNamespace(CLUSTER_NAMESPACE).withName(kr.getMetadata().getName()).get();
        return kcrao.reconcileRebalance(new Reconciliation("test-trigger", KafkaRebalance.RESOURCE_KIND, CLUSTER_NAMESPACE, kr.getMetadata().getName()), kr1);
    }).onComplete(context.succeeding(v -> {
        // the resource moved from PendingProposal to ProposalReady
        assertState(context, client, CLUSTER_NAMESPACE, kr.getMetadata().getName(), KafkaRebalanceState.ProposalReady);
        context.verify(() -> {
            KafkaRebalanceStatus rebalanceStatus = Crds.kafkaRebalanceOperation(client).inNamespace(CLUSTER_NAMESPACE).withName(kr.getMetadata().getName()).get().getStatus();
            assertTrue(rebalanceStatus.getOptimizationResult().size() > 0);
            assertNotNull(rebalanceStatus.getSessionId());
        });
        checkpoint.flag();
    }));
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) URISyntaxException(java.net.URISyntaxException) Annotations(io.strimzi.operator.common.Annotations) KafkaRebalanceSpecBuilder(io.strimzi.api.kafka.model.KafkaRebalanceSpecBuilder) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) AfterAll(org.junit.jupiter.api.AfterAll) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Collections.singleton(java.util.Collections.singleton) EnableKubernetesMockClient(io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient) BeforeAll(org.junit.jupiter.api.BeforeAll) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) ResourceUtils(io.strimzi.operator.cluster.ResourceUtils) MockKube2(io.strimzi.test.mockkube2.MockKube2) KafkaRebalance(io.strimzi.api.kafka.model.KafkaRebalance) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) SecretOperator(io.strimzi.operator.common.operator.resource.SecretOperator) VertxExtension(io.vertx.junit5.VertxExtension) KafkaRebalanceList(io.strimzi.api.kafka.KafkaRebalanceList) KafkaRebalanceAnnotation(io.strimzi.api.kafka.model.balancing.KafkaRebalanceAnnotation) Future(io.vertx.core.Future) NoSuchResourceException(io.strimzi.operator.cluster.model.NoSuchResourceException) ClientAndServer(org.mockserver.integration.ClientAndServer) CruiseControlResources(io.strimzi.api.kafka.model.CruiseControlResources) Test(org.junit.jupiter.api.Test) Labels(io.strimzi.operator.common.model.Labels) CruiseControlApi(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApi) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Secret(io.fabric8.kubernetes.api.model.Secret) Optional(java.util.Optional) Checkpoint(io.vertx.junit5.Checkpoint) Matchers.is(org.hamcrest.Matchers.is) Condition(io.strimzi.api.kafka.model.status.Condition) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) VertxTestContext(io.vertx.junit5.VertxTestContext) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) KafkaList(io.strimzi.api.kafka.KafkaList) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) KafkaRebalanceState(io.strimzi.api.kafka.model.balancing.KafkaRebalanceState) KafkaRebalanceBuilder(io.strimzi.api.kafka.model.KafkaRebalanceBuilder) CruiseControlApiImpl(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApiImpl) Crds(io.strimzi.api.kafka.Crds) KafkaRebalanceSpec(io.strimzi.api.kafka.model.KafkaRebalanceSpec) KafkaBuilder(io.strimzi.api.kafka.model.KafkaBuilder) KafkaRebalanceMode(io.strimzi.api.kafka.model.balancing.KafkaRebalanceMode) CruiseControlEndpoints(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlEndpoints) KafkaVersionTestUtils(io.strimzi.operator.cluster.KafkaVersionTestUtils) ConfigMapOperator(io.strimzi.operator.common.operator.resource.ConfigMapOperator) CrdOperator(io.strimzi.operator.common.operator.resource.CrdOperator) TestUtils(io.strimzi.test.TestUtils) ReconcileResult(io.strimzi.operator.common.operator.resource.ReconcileResult) ConnectException(java.net.ConnectException) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) NoStackTraceTimeoutException(io.strimzi.operator.common.operator.resource.NoStackTraceTimeoutException) CruiseControl(io.strimzi.operator.cluster.model.CruiseControl) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) Vertx(io.vertx.core.Vertx) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap) KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) Reconciliation(io.strimzi.operator.common.Reconciliation) AfterEach(org.junit.jupiter.api.AfterEach) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Kafka(io.strimzi.api.kafka.model.Kafka) CruiseControlRestException(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRestException) MockCruiseControl(io.strimzi.operator.cluster.operator.resource.cruisecontrol.MockCruiseControl) Collections(java.util.Collections) Checkpoint(io.vertx.junit5.Checkpoint) KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) Reconciliation(io.strimzi.operator.common.Reconciliation) KafkaRebalance(io.strimzi.api.kafka.model.KafkaRebalance)

Example 8 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi-kafka-operator by strimzi.

the class CruiseControlST method testCruiseControlIntraBrokerBalancing.

@ParallelNamespaceTest
@KRaftNotSupported("JBOD is not supported by KRaft mode and is used in this test case.")
void testCruiseControlIntraBrokerBalancing(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext);
    String diskSize = "6Gi";
    JbodStorage jbodStorage = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withDeleteClaim(true).withId(0).withSize(diskSize).build(), new PersistentClaimStorageBuilder().withDeleteClaim(true).withId(1).withSize(diskSize).build()).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaWithCruiseControl(testStorage.getClusterName(), 3, 3).editMetadata().withNamespace(testStorage.getNamespaceName()).endMetadata().editOrNewSpec().editKafka().withStorage(jbodStorage).endKafka().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaRebalanceTemplates.kafkaRebalance(testStorage.getClusterName()).editMetadata().withNamespace(testStorage.getNamespaceName()).endMetadata().editOrNewSpec().withRebalanceDisk(true).endSpec().build());
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(testStorage.getNamespaceName(), testStorage.getClusterName(), KafkaRebalanceState.ProposalReady);
    LOGGER.info("Checking status of KafkaRebalance");
    // The provision status should be "UNDECIDED" when doing an intra-broker disk balance because it is irrelevant to the provision status
    KafkaRebalanceStatus kafkaRebalanceStatus = KafkaRebalanceResource.kafkaRebalanceClient().inNamespace(testStorage.getNamespaceName()).withName(testStorage.getClusterName()).get().getStatus();
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("provisionStatus").toString(), containsString("UNDECIDED"));
}
Also used : JbodStorageBuilder(io.strimzi.api.kafka.model.storage.JbodStorageBuilder) KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) PersistentClaimStorageBuilder(io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder) TestStorage(io.strimzi.systemtest.storage.TestStorage) Matchers.containsString(org.hamcrest.Matchers.containsString) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 9 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi-kafka-operator by strimzi.

the class CruiseControlST method testCruiseControlTopicExclusion.

@ParallelNamespaceTest
@KRaftNotSupported("TopicOperator is not supported by KRaft mode and is used in this test class")
void testCruiseControlTopicExclusion(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String excludedTopic1 = "excluded-topic-1";
    final String excludedTopic2 = "excluded-topic-2";
    final String includedTopic = "included-topic";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaWithCruiseControl(clusterName, 3, 1).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, excludedTopic1, namespaceName).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, excludedTopic2, namespaceName).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, includedTopic, namespaceName).build());
    resourceManager.createResource(extensionContext, KafkaRebalanceTemplates.kafkaRebalance(clusterName).editOrNewSpec().withExcludedTopics("excluded-.*").endSpec().build());
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.ProposalReady);
    LOGGER.info("Checking status of KafkaRebalance");
    KafkaRebalanceStatus kafkaRebalanceStatus = KafkaRebalanceResource.kafkaRebalanceClient().inNamespace(namespaceName).withName(clusterName).get().getStatus();
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("excludedTopics").toString(), containsString(excludedTopic1));
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("excludedTopics").toString(), containsString(excludedTopic2));
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("excludedTopics").toString(), not(containsString(includedTopic)));
    KafkaRebalanceUtils.annotateKafkaRebalanceResource(new Reconciliation("test", KafkaRebalance.RESOURCE_KIND, namespace, clusterName), namespaceName, clusterName, KafkaRebalanceAnnotation.approve);
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.Ready);
}
Also used : KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) Reconciliation(io.strimzi.operator.common.Reconciliation) Matchers.containsString(org.hamcrest.Matchers.containsString) KRaftNotSupported(io.strimzi.systemtest.annotations.KRaftNotSupported) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 10 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method reconcile.

private Future<Void> reconcile(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, KafkaRebalanceState currentState, KafkaRebalanceAnnotation rebalanceAnnotation) {
    LOGGER.infoCr(reconciliation, "Rebalance action from state [{}]", currentState);
    if (Annotations.isReconciliationPausedWithAnnotation(kafkaRebalance)) {
        // we need to do this check again because it was triggered by a watcher
        KafkaRebalanceStatus status = new KafkaRebalanceStatus();
        Set<Condition> unknownAndDeprecatedConditions = validate(reconciliation, kafkaRebalance);
        unknownAndDeprecatedConditions.add(StatusUtils.getPausedCondition());
        status.setConditions(new ArrayList<>(unknownAndDeprecatedConditions));
        return updateStatus(reconciliation, kafkaRebalance, status, null).compose(i -> Future.succeededFuture());
    }
    if (kafkaRebalance.getSpec().isRebalanceDisk() && !usingJbodStorage) {
        String error = "Cannot set rebalanceDisk=true for Kafka clusters with a non-JBOD storage config. " + "Intra-broker balancing only applies to Kafka deployments that use JBOD storage with multiple disks.";
        LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", new InvalidResourceException(error).getMessage());
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException(error)).mapEmpty();
    }
    try {
        AbstractRebalanceOptions.AbstractRebalanceOptionsBuilder<?, ?> rebalanceOptionsBuilder = convertRebalanceSpecToRebalanceOptions(kafkaRebalance.getSpec());
        return computeNextStatus(reconciliation, host, apiClient, kafkaRebalance, currentState, rebalanceAnnotation, rebalanceOptionsBuilder).compose(desiredStatusAndMap -> {
            // do a new get to retrieve the current resource state.
            return kafkaRebalanceOperator.getAsync(reconciliation.namespace(), reconciliation.name()).compose(currentKafkaRebalance -> {
                if (currentKafkaRebalance != null) {
                    return configMapOperator.reconcile(reconciliation, kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName(), desiredStatusAndMap.getLoadMap()).compose(i -> updateStatus(reconciliation, currentKafkaRebalance, desiredStatusAndMap.getStatus(), null)).compose(updatedKafkaRebalance -> {
                        String message = "State updated to [{}] ";
                        if (rawRebalanceAnnotation(updatedKafkaRebalance) == null) {
                            LOGGER.infoCr(reconciliation, message + "and annotation {} is not set ", rebalanceStateConditionType(updatedKafkaRebalance.getStatus()), ANNO_STRIMZI_IO_REBALANCE);
                        } else {
                            LOGGER.infoCr(reconciliation, message + "with annotation {}={} ", rebalanceStateConditionType(updatedKafkaRebalance.getStatus()), ANNO_STRIMZI_IO_REBALANCE, rawRebalanceAnnotation(updatedKafkaRebalance));
                        }
                        if (hasRebalanceAnnotation(updatedKafkaRebalance)) {
                            if (currentState != KafkaRebalanceState.ReconciliationPaused && rebalanceAnnotation != KafkaRebalanceAnnotation.none && !currentState.isValidateAnnotation(rebalanceAnnotation)) {
                                return Future.succeededFuture();
                            } else {
                                LOGGER.infoCr(reconciliation, "Removing annotation {}={}", ANNO_STRIMZI_IO_REBALANCE, rawRebalanceAnnotation(updatedKafkaRebalance));
                                // Updated KafkaRebalance has rebalance annotation removed as
                                // action specified by user has been completed.
                                KafkaRebalance patchedKafkaRebalance = new KafkaRebalanceBuilder(updatedKafkaRebalance).editMetadata().removeFromAnnotations(ANNO_STRIMZI_IO_REBALANCE).endMetadata().build();
                                return kafkaRebalanceOperator.patchAsync(reconciliation, patchedKafkaRebalance);
                            }
                        } else {
                            LOGGER.debugCr(reconciliation, "No annotation {}", ANNO_STRIMZI_IO_REBALANCE);
                            return Future.succeededFuture();
                        }
                    }).mapEmpty();
                } else {
                    return Future.succeededFuture();
                }
            }, exception -> {
                LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", exception.getMessage());
                return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty();
            });
        }, exception -> {
            LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", exception.getMessage());
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty();
        });
    } catch (IllegalArgumentException e) {
        LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", e.getMessage());
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), e).mapEmpty();
    }
}
Also used : Condition(io.strimzi.api.kafka.model.status.Condition) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) Arrays(java.util.Arrays) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) ANNO_STRIMZI_IO_REBALANCE(io.strimzi.operator.common.Annotations.ANNO_STRIMZI_IO_REBALANCE) Watcher(io.fabric8.kubernetes.client.Watcher) Annotations(io.strimzi.operator.common.Annotations) CruiseControlConfiguration(io.strimzi.operator.cluster.model.CruiseControlConfiguration) AbstractRebalanceOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.AbstractRebalanceOptions) Resource(io.fabric8.kubernetes.client.dsl.Resource) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CruiseControlRebalanceResponse(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRebalanceResponse) Map(java.util.Map) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) JsonObject(io.vertx.core.json.JsonObject) CruiseControlRebalanceKeys(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRebalanceKeys) ModelUtils(io.strimzi.operator.cluster.model.ModelUtils) KubernetesClientException(io.fabric8.kubernetes.client.KubernetesClientException) AbstractOperator(io.strimzi.operator.common.AbstractOperator) StatusUtils(io.strimzi.operator.common.operator.resource.StatusUtils) KafkaRebalance(io.strimzi.api.kafka.model.KafkaRebalance) SecretOperator(io.strimzi.operator.common.operator.resource.SecretOperator) Set(java.util.Set) KafkaRebalanceList(io.strimzi.api.kafka.KafkaRebalanceList) KafkaRebalanceAnnotation(io.strimzi.api.kafka.model.balancing.KafkaRebalanceAnnotation) Future(io.vertx.core.Future) Collectors(java.util.stream.Collectors) NoSuchResourceException(io.strimzi.operator.cluster.model.NoSuchResourceException) CruiseControlUserTaskStatus(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlUserTaskStatus) KafkaRebalanceStatusBuilder(io.strimzi.api.kafka.model.status.KafkaRebalanceStatusBuilder) CruiseControlResources(io.strimzi.api.kafka.model.CruiseControlResources) AddBrokerOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.AddBrokerOptions) List(java.util.List) Labels(io.strimzi.operator.common.model.Labels) Stream(java.util.stream.Stream) CruiseControlApi(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApi) Secret(io.fabric8.kubernetes.api.model.Secret) Optional(java.util.Optional) Condition(io.strimzi.api.kafka.model.status.Condition) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) KafkaList(io.strimzi.api.kafka.KafkaList) KafkaRebalanceState(io.strimzi.api.kafka.model.balancing.KafkaRebalanceState) KafkaRebalanceBuilder(io.strimzi.api.kafka.model.KafkaRebalanceBuilder) HashMap(java.util.HashMap) CruiseControlApiImpl(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApiImpl) KafkaRebalanceSpec(io.strimzi.api.kafka.model.KafkaRebalanceSpec) ArrayList(java.util.ArrayList) KafkaRebalanceMode(io.strimzi.api.kafka.model.balancing.KafkaRebalanceMode) HashSet(java.util.HashSet) WatcherException(io.fabric8.kubernetes.client.WatcherException) CompositeFuture(io.vertx.core.CompositeFuture) ConfigMapOperator(io.strimzi.operator.common.operator.resource.ConfigMapOperator) CrdOperator(io.strimzi.operator.common.operator.resource.CrdOperator) RebalanceOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.RebalanceOptions) CruiseControl(io.strimzi.operator.cluster.model.CruiseControl) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage) ReconciliationLogger(io.strimzi.operator.common.ReconciliationLogger) InvalidResourceException(io.strimzi.operator.cluster.model.InvalidResourceException) Promise(io.vertx.core.Promise) Vertx(io.vertx.core.Vertx) CruiseControlLoadParameters(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlLoadParameters) ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap) KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder) Reconciliation(io.strimzi.operator.common.Reconciliation) JsonArray(io.vertx.core.json.JsonArray) HTTP_DEFAULT_IDLE_TIMEOUT_SECONDS(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApiImpl.HTTP_DEFAULT_IDLE_TIMEOUT_SECONDS) StatusDiff(io.strimzi.operator.cluster.model.StatusDiff) Util(io.strimzi.operator.common.Util) RemoveBrokerOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.RemoveBrokerOptions) AbstractWatchableStatusedResourceOperator(io.strimzi.operator.common.operator.resource.AbstractWatchableStatusedResourceOperator) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Kafka(io.strimzi.api.kafka.model.Kafka) CruiseControlRestException(io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlRestException) Collections(java.util.Collections) KafkaRebalanceStatus(io.strimzi.api.kafka.model.status.KafkaRebalanceStatus) AbstractRebalanceOptions(io.strimzi.operator.cluster.operator.resource.cruisecontrol.AbstractRebalanceOptions) KafkaRebalanceBuilder(io.strimzi.api.kafka.model.KafkaRebalanceBuilder) KafkaRebalance(io.strimzi.api.kafka.model.KafkaRebalance)

Aggregations

KafkaRebalanceStatus (io.strimzi.api.kafka.model.status.KafkaRebalanceStatus)22 Secret (io.fabric8.kubernetes.api.model.Secret)12 KafkaRebalanceBuilder (io.strimzi.api.kafka.model.KafkaRebalanceBuilder)12 KafkaRebalanceAnnotation (io.strimzi.api.kafka.model.balancing.KafkaRebalanceAnnotation)12 KafkaRebalanceState (io.strimzi.api.kafka.model.balancing.KafkaRebalanceState)12 Condition (io.strimzi.api.kafka.model.status.Condition)12 InvalidResourceException (io.strimzi.operator.cluster.model.InvalidResourceException)12 NoSuchResourceException (io.strimzi.operator.cluster.model.NoSuchResourceException)12 CruiseControlApi (io.strimzi.operator.cluster.operator.resource.cruisecontrol.CruiseControlApi)12 Reconciliation (io.strimzi.operator.common.Reconciliation)12 ConfigMap (io.fabric8.kubernetes.api.model.ConfigMap)10 KubernetesClient (io.fabric8.kubernetes.client.KubernetesClient)10 KafkaList (io.strimzi.api.kafka.KafkaList)10 KafkaRebalanceList (io.strimzi.api.kafka.KafkaRebalanceList)10 CruiseControlResources (io.strimzi.api.kafka.model.CruiseControlResources)10 Kafka (io.strimzi.api.kafka.model.Kafka)10 KafkaRebalance (io.strimzi.api.kafka.model.KafkaRebalance)10 KafkaRebalanceSpec (io.strimzi.api.kafka.model.KafkaRebalanceSpec)10 JbodStorage (io.strimzi.api.kafka.model.storage.JbodStorage)10 ClusterOperatorConfig (io.strimzi.operator.cluster.ClusterOperatorConfig)10