Examples with KafkaRebalanceStatus io.strimzi.api.kafka.model.status.KafkaRebalanceStatus used on opensource projects

Example 1 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperatorTest method testRebalanceStatusInProposalReadyState.

/**
 * Test KafkaRebalance status in ProposalReady state
 *
 * 1. KafkaRebalance resource is created; it is in the 'New' state
 * 2. KafkaRebalance go through the `PendingProposal` to `ProposalReady` state
 * 3. KafkaRebalance status should contain optimization result and session id
 */
@Test
public void testRebalanceStatusInProposalReadyState(VertxTestContext context) throws IOException, URISyntaxException {
    MockCruiseControl.setupCCRebalanceResponse(ccServer, 2);
    KafkaRebalance kr = createKafkaRebalance(CLUSTER_NAMESPACE, CLUSTER_NAME, RESOURCE_NAME, new KafkaRebalanceSpecBuilder().build());
    Crds.kafkaRebalanceOperation(kubernetesClient).inNamespace(CLUSTER_NAMESPACE).create(kr);
    when(mockKafkaOps.getAsync(CLUSTER_NAMESPACE, CLUSTER_NAME)).thenReturn(Future.succeededFuture(kafka));
    mockSecretResources();
    mockRebalanceOperator(mockRebalanceOps, mockCmOps, CLUSTER_NAMESPACE, RESOURCE_NAME, kubernetesClient);
    Checkpoint checkpoint = context.checkpoint();
    kcrao.reconcileRebalance(new Reconciliation("test-trigger", KafkaRebalance.RESOURCE_KIND, CLUSTER_NAMESPACE, RESOURCE_NAME), kr).onComplete(context.succeeding(v -> {
        assertState(context, kubernetesClient, CLUSTER_NAMESPACE, RESOURCE_NAME, KafkaRebalanceState.PendingProposal);
    })).compose(v -> {
        // trigger another reconcile to process the PendingProposal state
        KafkaRebalance kr1 = Crds.kafkaRebalanceOperation(kubernetesClient).inNamespace(CLUSTER_NAMESPACE).withName(RESOURCE_NAME).get();
        return kcrao.reconcileRebalance(new Reconciliation("test-trigger", KafkaRebalance.RESOURCE_KIND, CLUSTER_NAMESPACE, RESOURCE_NAME), kr1);
    }).onComplete(context.succeeding(v -> {
        assertState(context, kubernetesClient, CLUSTER_NAMESPACE, RESOURCE_NAME, KafkaRebalanceState.ProposalReady);
        context.verify(() -> {
            KafkaRebalanceStatus rebalanceStatus = Crds.kafkaRebalanceOperation(kubernetesClient).inNamespace(CLUSTER_NAMESPACE).withName(RESOURCE_NAME).get().getStatus();
            assertTrue(rebalanceStatus.getOptimizationResult().size() > 0);
            assertNotNull(rebalanceStatus.getSessionId());
        });
        checkpoint.flag();
    }));
}

Example 2 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class CruiseControlST method testCruiseControlTopicExclusion.

@ParallelNamespaceTest
void testCruiseControlTopicExclusion(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String excludedTopic1 = "excluded-topic-1";
    final String excludedTopic2 = "excluded-topic-2";
    final String includedTopic = "included-topic";
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaWithCruiseControl(clusterName, 3, 3).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, excludedTopic1, namespaceName).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, excludedTopic2, namespaceName).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, includedTopic, namespaceName).build());
    resourceManager.createResource(extensionContext, KafkaRebalanceTemplates.kafkaRebalance(clusterName).editOrNewSpec().withExcludedTopics("excluded-.*").endSpec().build());
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.ProposalReady);
    LOGGER.info("Checking status of KafkaRebalance");
    KafkaRebalanceStatus kafkaRebalanceStatus = KafkaRebalanceResource.kafkaRebalanceClient().inNamespace(namespaceName).withName(clusterName).get().getStatus();
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("excludedTopics").toString(), containsString(excludedTopic1));
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("excludedTopics").toString(), containsString(excludedTopic2));
    assertThat(kafkaRebalanceStatus.getOptimizationResult().get("excludedTopics").toString(), not(containsString(includedTopic)));
    KafkaRebalanceUtils.annotateKafkaRebalanceResource(new Reconciliation("test", KafkaRebalance.RESOURCE_KIND, namespace, clusterName), namespaceName, clusterName, KafkaRebalanceAnnotation.approve);
    KafkaRebalanceUtils.waitForKafkaRebalanceCustomResourceState(namespaceName, clusterName, KafkaRebalanceState.Ready);
}

Example 3 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method reconcileRebalance.

/**
 * Reconcile loop for the KafkaRebalance
 */
/* test */
Future<Void> reconcileRebalance(Reconciliation reconciliation, KafkaRebalance kafkaRebalance) {
    if (kafkaRebalance == null) {
        LOGGER.infoCr(reconciliation, "Rebalance resource deleted");
        return Future.succeededFuture();
    }
    String clusterName = kafkaRebalance.getMetadata().getLabels() == null ? null : kafkaRebalance.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL);
    String clusterNamespace = kafkaRebalance.getMetadata().getNamespace();
    if (clusterName == null) {
        LOGGER.warnCr(reconciliation, "Resource lacks label '{}': No cluster related to a possible rebalance.", Labels.STRIMZI_CLUSTER_LABEL);
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException("Resource lacks label '" + Labels.STRIMZI_CLUSTER_LABEL + "': No cluster related to a possible rebalance.")).mapEmpty();
    }
    // Get associated Kafka cluster state
    return kafkaOperator.getAsync(clusterNamespace, clusterName).compose(kafka -> {
        if (kafka == null) {
            LOGGER.warnCr(reconciliation, "Kafka resource '{}' identified by label '{}' does not exist in namespace {}.", clusterName, Labels.STRIMZI_CLUSTER_LABEL, clusterNamespace);
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new NoSuchResourceException("Kafka resource '" + clusterName + "' identified by label '" + Labels.STRIMZI_CLUSTER_LABEL + "' does not exist in namespace " + clusterNamespace + ".")).mapEmpty();
        } else if (!Util.matchesSelector(kafkaSelector, kafka)) {
            LOGGER.debugCr(reconciliation, "{} {} in namespace {} belongs to a Kafka cluster {} which does not match label selector {} and will be ignored", kind(), kafkaRebalance.getMetadata().getName(), clusterNamespace, clusterName, kafkaSelector.get().getMatchLabels());
            return Future.succeededFuture();
        } else if (kafka.getSpec().getCruiseControl() == null) {
            LOGGER.warnCr(reconciliation, "Kafka resource lacks 'cruiseControl' declaration : No deployed Cruise Control for doing a rebalance.");
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException("Kafka resource lacks 'cruiseControl' declaration " + ": No deployed Cruise Control for doing a rebalance.")).mapEmpty();
        }
        if (kafka.getSpec().getKafka().getStorage() instanceof JbodStorage) {
            usingJbodStorage = true;
        }
        String ccSecretName = CruiseControlResources.secretName(clusterName);
        String ccApiSecretName = CruiseControlResources.apiSecretName(clusterName);
        Future<Secret> ccSecretFuture = secretOperations.getAsync(clusterNamespace, ccSecretName);
        Future<Secret> ccApiSecretFuture = secretOperations.getAsync(clusterNamespace, ccApiSecretName);
        return CompositeFuture.join(ccSecretFuture, ccApiSecretFuture).compose(compositeFuture -> {
            Secret ccSecret = compositeFuture.resultAt(0);
            if (ccSecret == null) {
                return Future.failedFuture(Util.missingSecretException(clusterNamespace, ccSecretName));
            }
            Secret ccApiSecret = compositeFuture.resultAt(1);
            if (ccApiSecret == null) {
                return Future.failedFuture(Util.missingSecretException(clusterNamespace, ccApiSecretName));
            }
            CruiseControlConfiguration c = new CruiseControlConfiguration(reconciliation, kafka.getSpec().getCruiseControl().getConfig().entrySet());
            boolean apiAuthEnabled = CruiseControl.isApiAuthEnabled(c);
            boolean apiSslEnabled = CruiseControl.isApiSslEnabled(c);
            CruiseControlApi apiClient = cruiseControlClientProvider(ccSecret, ccApiSecret, apiAuthEnabled, apiSslEnabled);
            // get latest KafkaRebalance state as it may have changed
            return kafkaRebalanceOperator.getAsync(kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName()).compose(currentKafkaRebalance -> {
                KafkaRebalanceStatus kafkaRebalanceStatus = currentKafkaRebalance.getStatus();
                KafkaRebalanceState currentState;
                // cluster rebalance is new or it is in one of the others states
                if (kafkaRebalanceStatus == null || kafkaRebalanceStatus.getConditions().stream().filter(cond -> "ReconciliationPaused".equals(cond.getType())).findAny().isPresent()) {
                    currentState = KafkaRebalanceState.New;
                } else {
                    String rebalanceStateType = rebalanceStateConditionType(kafkaRebalanceStatus);
                    if (rebalanceStateType == null) {
                        throw new RuntimeException("Unable to find KafkaRebalance State in current KafkaRebalance status");
                    }
                    currentState = KafkaRebalanceState.valueOf(rebalanceStateType);
                }
                // Check annotation
                KafkaRebalanceAnnotation rebalanceAnnotation = rebalanceAnnotation(reconciliation, currentKafkaRebalance);
                return reconcile(reconciliation, cruiseControlHost(clusterName, clusterNamespace), apiClient, currentKafkaRebalance, currentState, rebalanceAnnotation).mapEmpty();
            }, exception -> Future.failedFuture(exception).mapEmpty());
        });
    }, exception -> updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty());
}

Example 4 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method onRebalancing.

/**
 * This method handles the transition from {@code Rebalancing} state.
 * It starts a periodic timer in order to check the status of the ongoing rebalance processing on Cruise Control side.
 * In order to do that, it calls the related Cruise Control REST API about asking the user task status.
 * When the rebalance is finished, the next state is {@code Ready}.
 * If the user sets the strimzi.io/rebalance annotation to 'stop', it calls the Cruise Control REST API for stopping the ongoing task
 * and then transitions to the {@code Stopped} state.
 * If the user sets any other values for the strimzi.io/rebalance annotation, it is just ignored and the user task checks continue.
 * This method holds the lock until the rebalance is finished, the ongoing task is stopped or any exception is raised.
 *
 * @param reconciliation Reconciliation information
 * @param host Cruise Control service to which sending the REST API requests
 * @param apiClient Cruise Control REST API client instance
 * @param kafkaRebalance Current {@code KafkaRebalance} resource
 * @param rebalanceAnnotation The current value for the strimzi.io/rebalance annotation
 * @return a Future with the next {@code MapAndStatus<ConfigMap, KafkaRebalanceStatus>} including the state
 */
private Future<MapAndStatus<ConfigMap, KafkaRebalanceStatus>> onRebalancing(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, KafkaRebalanceAnnotation rebalanceAnnotation) {
    Promise<MapAndStatus<ConfigMap, KafkaRebalanceStatus>> p = Promise.promise();
    if (rebalanceAnnotation == KafkaRebalanceAnnotation.none) {
        LOGGER.infoCr(reconciliation, "Starting Cruise Control rebalance user task status timer");
        String sessionId = kafkaRebalance.getStatus().getSessionId();
        AtomicInteger ccApiErrorCount = new AtomicInteger();
        vertx.setPeriodic(REBALANCE_POLLING_TIMER_MS, t -> {
            // Check that we have not already failed to contact the API beyond the allowed number of times.
            if (ccApiErrorCount.get() >= MAX_API_RETRIES) {
                vertx.cancelTimer(t);
                p.fail(new CruiseControlRestException("Unable to reach Cruise Control API after " + MAX_API_RETRIES + " attempts"));
            }
            kafkaRebalanceOperator.getAsync(kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName()).onSuccess(currentKafkaRebalance -> {
                // Checking that the resource was not deleted between periodic polls
                if (currentKafkaRebalance != null) {
                    // Safety check as timer might be called again (from a delayed timer firing)
                    if (state(currentKafkaRebalance) == KafkaRebalanceState.Rebalancing) {
                        if (rebalanceAnnotation(reconciliation, currentKafkaRebalance) == KafkaRebalanceAnnotation.stop) {
                            LOGGER.debugCr(reconciliation, "Stopping current Cruise Control rebalance user task");
                            vertx.cancelTimer(t);
                            apiClient.stopExecution(host, CruiseControl.REST_API_PORT).onSuccess(r -> p.complete(buildRebalanceStatus(null, KafkaRebalanceState.Stopped, validate(reconciliation, kafkaRebalance)))).onFailure(e -> {
                                LOGGER.errorCr(reconciliation, "Cruise Control stopping execution failed", e.getCause());
                                p.fail(e.getCause());
                            });
                        } else {
                            LOGGER.infoCr(reconciliation, "Getting Cruise Control rebalance user task status");
                            apiClient.getUserTaskStatus(host, CruiseControl.REST_API_PORT, sessionId).onSuccess(cruiseControlResponse -> {
                                JsonObject taskStatusJson = cruiseControlResponse.getJson();
                                CruiseControlUserTaskStatus taskStatus = CruiseControlUserTaskStatus.lookup(taskStatusJson.getString("Status"));
                                switch(taskStatus) {
                                    case COMPLETED:
                                        vertx.cancelTimer(t);
                                        LOGGER.infoCr(reconciliation, "Rebalance ({}) is now complete", sessionId);
                                        p.complete(buildRebalanceStatus(kafkaRebalance, null, KafkaRebalanceState.Ready, taskStatusJson, validate(reconciliation, kafkaRebalance)));
                                        break;
                                    case COMPLETED_WITH_ERROR:
                                        // TODO: There doesn't seem to be a way to retrieve the actual error message from the user tasks endpoint?
                                        // We may need to propose an upstream PR for this.
                                        // TODO: Once we can get the error details we need to add an error field to the Rebalance Status to hold
                                        // details of any issues while rebalancing.
                                        LOGGER.errorCr(reconciliation, "Rebalance ({}) optimization proposal has failed to complete", sessionId);
                                        vertx.cancelTimer(t);
                                        p.complete(buildRebalanceStatus(sessionId, KafkaRebalanceState.NotReady, validate(reconciliation, kafkaRebalance)));
                                        break;
                                    case // Rebalance is still in progress
                                    IN_EXECUTION:
                                        // the proposal is complete but the optimisation proposal summary will be missing.
                                        if (currentKafkaRebalance.getStatus().getOptimizationResult() == null || currentKafkaRebalance.getStatus().getOptimizationResult().isEmpty()) {
                                            LOGGER.infoCr(reconciliation, "Rebalance ({}) optimization proposal is now ready and has been added to the status", sessionId);
                                            // Cancel the timer so that the status is returned and updated.
                                            vertx.cancelTimer(t);
                                            p.complete(buildRebalanceStatus(kafkaRebalance, sessionId, KafkaRebalanceState.Rebalancing, taskStatusJson, validate(reconciliation, kafkaRebalance)));
                                        }
                                        ccApiErrorCount.set(0);
                                        // We can then update the status at this point.
                                        break;
                                    case // Rebalance proposal is still being calculated
                                    ACTIVE:
                                        // If a rebalance(dryrun=false) was called and the proposal is still being prepared then the task
                                        // will be in an ACTIVE state. When the proposal is ready it will shift to IN_EXECUTION and we will
                                        // check that the optimisation proposal is added to the status on the next reconcile.
                                        LOGGER.infoCr(reconciliation, "Rebalance ({}) optimization proposal is still being prepared", sessionId);
                                        ccApiErrorCount.set(0);
                                        break;
                                    default:
                                        LOGGER.errorCr(reconciliation, "Unexpected state {}", taskStatus);
                                        vertx.cancelTimer(t);
                                        p.fail("Unexpected state " + taskStatus);
                                        break;
                                }
                            }).onFailure(e -> {
                                LOGGER.errorCr(reconciliation, "Cruise Control getting rebalance task status failed", e.getCause());
                                // To make sure this error is not just a temporary problem with the network we retry several times.
                                // If the number of errors pass the MAX_API_ERRORS limit then the period method will fail the promise.
                                ccApiErrorCount.getAndIncrement();
                            });
                        }
                    } else {
                        p.complete(new MapAndStatus<>(null, currentKafkaRebalance.getStatus()));
                    }
                } else {
                    LOGGER.debugCr(reconciliation, "Rebalance resource was deleted, stopping the request time");
                    vertx.cancelTimer(t);
                    p.complete();
                }
            }).onFailure(e -> {
                LOGGER.errorCr(reconciliation, "Cruise Control getting rebalance resource failed", e.getCause());
                vertx.cancelTimer(t);
                p.fail(e.getCause());
            });
        });
    } else {
        p.complete(new MapAndStatus<>(null, kafkaRebalance.getStatus()));
    }
    return p.future();
}

Example 5 with KafkaRebalanceStatus

use of io.strimzi.api.kafka.model.status.KafkaRebalanceStatus in project strimzi-kafka-operator by strimzi.

the class KafkaRebalanceAssemblyOperatorTest method testRebalanceStatusInProposalReadyState.

/**
 * Test KafkaRebalance status in ProposalReady state
 *
 * 1. KafkaRebalance resource is created; it is in the 'New' state
 * 2. KafkaRebalance go through the `PendingProposal` to `ProposalReady` state
 * 3. KafkaRebalance status should contain optimization result and session id
 */
@Test
public void testRebalanceStatusInProposalReadyState(VertxTestContext context) throws IOException, URISyntaxException {
    MockCruiseControl.setupCCRebalanceResponse(ccServer, 2);
    KafkaRebalance kr = createKafkaRebalance(CLUSTER_NAMESPACE, CLUSTER_NAME, RESOURCE_NAME, new KafkaRebalanceSpecBuilder().build());
    Crds.kafkaRebalanceOperation(kubernetesClient).inNamespace(CLUSTER_NAMESPACE).create(kr);
    when(mockKafkaOps.getAsync(CLUSTER_NAMESPACE, CLUSTER_NAME)).thenReturn(Future.succeededFuture(kafka));
    mockSecretResources();
    mockRebalanceOperator(mockRebalanceOps, mockCmOps, CLUSTER_NAMESPACE, RESOURCE_NAME, kubernetesClient);
    Checkpoint checkpoint = context.checkpoint();
    kcrao.reconcileRebalance(new Reconciliation("test-trigger", KafkaRebalance.RESOURCE_KIND, CLUSTER_NAMESPACE, RESOURCE_NAME), kr).onComplete(context.succeeding(v -> {
        assertState(context, kubernetesClient, CLUSTER_NAMESPACE, RESOURCE_NAME, KafkaRebalanceState.PendingProposal);
    })).compose(v -> {
        // trigger another reconcile to process the PendingProposal state
        KafkaRebalance kr1 = Crds.kafkaRebalanceOperation(kubernetesClient).inNamespace(CLUSTER_NAMESPACE).withName(RESOURCE_NAME).get();
        return kcrao.reconcileRebalance(new Reconciliation("test-trigger", KafkaRebalance.RESOURCE_KIND, CLUSTER_NAMESPACE, RESOURCE_NAME), kr1);
    }).onComplete(context.succeeding(v -> {
        assertState(context, kubernetesClient, CLUSTER_NAMESPACE, RESOURCE_NAME, KafkaRebalanceState.ProposalReady);
        context.verify(() -> {
            KafkaRebalanceStatus rebalanceStatus = Crds.kafkaRebalanceOperation(kubernetesClient).inNamespace(CLUSTER_NAMESPACE).withName(RESOURCE_NAME).get().getStatus();
            assertTrue(rebalanceStatus.getOptimizationResult().size() > 0);
            assertNotNull(rebalanceStatus.getSessionId());
        });
        checkpoint.flag();
    }));
}