Examples with KafkaRebalanceState io.strimzi.api.kafka.model.balancing.KafkaRebalanceState used on opensource projects

Example 1 with KafkaRebalanceState

use of io.strimzi.api.kafka.model.balancing.KafkaRebalanceState in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method reconcileRebalance.

/**
 * Reconcile loop for the KafkaRebalance
 */
/* test */
Future<Void> reconcileRebalance(Reconciliation reconciliation, KafkaRebalance kafkaRebalance) {
    if (kafkaRebalance == null) {
        LOGGER.infoCr(reconciliation, "Rebalance resource deleted");
        return Future.succeededFuture();
    }
    String clusterName = kafkaRebalance.getMetadata().getLabels() == null ? null : kafkaRebalance.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL);
    String clusterNamespace = kafkaRebalance.getMetadata().getNamespace();
    if (clusterName == null) {
        LOGGER.warnCr(reconciliation, "Resource lacks label '{}': No cluster related to a possible rebalance.", Labels.STRIMZI_CLUSTER_LABEL);
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException("Resource lacks label '" + Labels.STRIMZI_CLUSTER_LABEL + "': No cluster related to a possible rebalance.")).mapEmpty();
    }
    // Get associated Kafka cluster state
    return kafkaOperator.getAsync(clusterNamespace, clusterName).compose(kafka -> {
        if (kafka == null) {
            LOGGER.warnCr(reconciliation, "Kafka resource '{}' identified by label '{}' does not exist in namespace {}.", clusterName, Labels.STRIMZI_CLUSTER_LABEL, clusterNamespace);
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new NoSuchResourceException("Kafka resource '" + clusterName + "' identified by label '" + Labels.STRIMZI_CLUSTER_LABEL + "' does not exist in namespace " + clusterNamespace + ".")).mapEmpty();
        } else if (!Util.matchesSelector(kafkaSelector, kafka)) {
            LOGGER.debugCr(reconciliation, "{} {} in namespace {} belongs to a Kafka cluster {} which does not match label selector {} and will be ignored", kind(), kafkaRebalance.getMetadata().getName(), clusterNamespace, clusterName, kafkaSelector.get().getMatchLabels());
            return Future.succeededFuture();
        } else if (kafka.getSpec().getCruiseControl() == null) {
            LOGGER.warnCr(reconciliation, "Kafka resource lacks 'cruiseControl' declaration : No deployed Cruise Control for doing a rebalance.");
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException("Kafka resource lacks 'cruiseControl' declaration " + ": No deployed Cruise Control for doing a rebalance.")).mapEmpty();
        }
        if (kafka.getSpec().getKafka().getStorage() instanceof JbodStorage) {
            usingJbodStorage = true;
        }
        String ccSecretName = CruiseControlResources.secretName(clusterName);
        String ccApiSecretName = CruiseControlResources.apiSecretName(clusterName);
        Future<Secret> ccSecretFuture = secretOperations.getAsync(clusterNamespace, ccSecretName);
        Future<Secret> ccApiSecretFuture = secretOperations.getAsync(clusterNamespace, ccApiSecretName);
        return CompositeFuture.join(ccSecretFuture, ccApiSecretFuture).compose(compositeFuture -> {
            Secret ccSecret = compositeFuture.resultAt(0);
            if (ccSecret == null) {
                return Future.failedFuture(Util.missingSecretException(clusterNamespace, ccSecretName));
            }
            Secret ccApiSecret = compositeFuture.resultAt(1);
            if (ccApiSecret == null) {
                return Future.failedFuture(Util.missingSecretException(clusterNamespace, ccApiSecretName));
            }
            CruiseControlConfiguration c = new CruiseControlConfiguration(reconciliation, kafka.getSpec().getCruiseControl().getConfig().entrySet());
            boolean apiAuthEnabled = CruiseControl.isApiAuthEnabled(c);
            boolean apiSslEnabled = CruiseControl.isApiSslEnabled(c);
            CruiseControlApi apiClient = cruiseControlClientProvider(ccSecret, ccApiSecret, apiAuthEnabled, apiSslEnabled);
            // get latest KafkaRebalance state as it may have changed
            return kafkaRebalanceOperator.getAsync(kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName()).compose(currentKafkaRebalance -> {
                KafkaRebalanceStatus kafkaRebalanceStatus = currentKafkaRebalance.getStatus();
                KafkaRebalanceState currentState;
                // cluster rebalance is new or it is in one of the others states
                if (kafkaRebalanceStatus == null || kafkaRebalanceStatus.getConditions().stream().filter(cond -> "ReconciliationPaused".equals(cond.getType())).findAny().isPresent()) {
                    currentState = KafkaRebalanceState.New;
                } else {
                    String rebalanceStateType = rebalanceStateConditionType(kafkaRebalanceStatus);
                    if (rebalanceStateType == null) {
                        throw new RuntimeException("Unable to find KafkaRebalance State in current KafkaRebalance status");
                    }
                    currentState = KafkaRebalanceState.valueOf(rebalanceStateType);
                }
                // Check annotation
                KafkaRebalanceAnnotation rebalanceAnnotation = rebalanceAnnotation(reconciliation, currentKafkaRebalance);
                return reconcile(reconciliation, cruiseControlHost(clusterName, clusterNamespace), apiClient, currentKafkaRebalance, currentState, rebalanceAnnotation).mapEmpty();
            }, exception -> Future.failedFuture(exception).mapEmpty());
        });
    }, exception -> updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty());
}

Example 2 with KafkaRebalanceState

use of io.strimzi.api.kafka.model.balancing.KafkaRebalanceState in project strimzi-kafka-operator by strimzi.

the class KafkaRebalanceAssemblyOperator method requestRebalance.

private Future<MapAndStatus<ConfigMap, KafkaRebalanceStatus>> requestRebalance(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, boolean dryrun, RebalanceOptions.RebalanceOptionsBuilder rebalanceOptionsBuilder, String userTaskID) {
    LOGGER.infoCr(reconciliation, "Requesting Cruise Control rebalance [dryrun={}]", dryrun);
    rebalanceOptionsBuilder.withVerboseResponse();
    if (!dryrun) {
        rebalanceOptionsBuilder.withFullRun();
    }
    return apiClient.rebalance(host, CruiseControl.REST_API_PORT, rebalanceOptionsBuilder.build(), userTaskID).map(response -> {
        if (dryrun) {
            if (response.isNotEnoughDataForProposal()) {
                // Need to re-request the proposal at a later time so move to the PendingProposal State.
                return buildRebalanceStatus(null, KafkaRebalanceState.PendingProposal, validate(reconciliation, kafkaRebalance));
            } else if (response.isProposalStillCalaculating()) {
                // with the corresponding session-id so we move to the PendingProposal State.
                return buildRebalanceStatus(response.getUserTaskId(), KafkaRebalanceState.PendingProposal, validate(reconciliation, kafkaRebalance));
            }
        } else {
            if (response.isNotEnoughDataForProposal()) {
                // this failed tasks (COMPLETED_WITH_ERROR)
                return buildRebalanceStatus(null, KafkaRebalanceState.PendingProposal, validate(reconciliation, kafkaRebalance));
            } else if (response.isProposalStillCalaculating()) {
                // In the onRebalancing method the optimization proposal will be added when it is ready.
                return buildRebalanceStatus(response.getUserTaskId(), KafkaRebalanceState.Rebalancing, validate(reconciliation, kafkaRebalance));
            }
        }
        if (response.getJson() != null && response.getJson().containsKey(CruiseControlRebalanceKeys.SUMMARY.getKey())) {
            // If there is enough data and the proposal is complete (the response has the "summary" key) then we move
            // to ProposalReady for a dry run or to the Rebalancing state for a full run
            KafkaRebalanceState ready = dryrun ? KafkaRebalanceState.ProposalReady : KafkaRebalanceState.Rebalancing;
            return buildRebalanceStatus(kafkaRebalance, response.getUserTaskId(), ready, response.getJson(), validate(reconciliation, kafkaRebalance));
        } else {
            throw new CruiseControlRestException("Rebalance returned unknown response: " + response.toString());
        }
    });
}

Example 3 with KafkaRebalanceState

use of io.strimzi.api.kafka.model.balancing.KafkaRebalanceState in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method requestRebalance.

private Future<MapAndStatus<ConfigMap, KafkaRebalanceStatus>> requestRebalance(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, boolean dryrun, RebalanceOptions.RebalanceOptionsBuilder rebalanceOptionsBuilder, String userTaskID) {
    LOGGER.infoCr(reconciliation, "Requesting Cruise Control rebalance [dryrun={}]", dryrun);
    rebalanceOptionsBuilder.withVerboseResponse();
    if (!dryrun) {
        rebalanceOptionsBuilder.withFullRun();
    }
    return apiClient.rebalance(host, CruiseControl.REST_API_PORT, rebalanceOptionsBuilder.build(), userTaskID).map(response -> {
        if (dryrun) {
            if (response.isNotEnoughDataForProposal()) {
                // Need to re-request the proposal at a later time so move to the PendingProposal State.
                return buildRebalanceStatus(null, KafkaRebalanceState.PendingProposal, validate(reconciliation, kafkaRebalance));
            } else if (response.isProposalStillCalaculating()) {
                // with the corresponding session-id so we move to the PendingProposal State.
                return buildRebalanceStatus(response.getUserTaskId(), KafkaRebalanceState.PendingProposal, validate(reconciliation, kafkaRebalance));
            }
        } else {
            if (response.isNotEnoughDataForProposal()) {
                // this failed tasks (COMPLETED_WITH_ERROR)
                return buildRebalanceStatus(null, KafkaRebalanceState.PendingProposal, validate(reconciliation, kafkaRebalance));
            } else if (response.isProposalStillCalaculating()) {
                // In the onRebalancing method the optimization proposal will be added when it is ready.
                return buildRebalanceStatus(response.getUserTaskId(), KafkaRebalanceState.Rebalancing, validate(reconciliation, kafkaRebalance));
            }
        }
        if (response.getJson() != null && response.getJson().containsKey(CruiseControlRebalanceKeys.SUMMARY.getKey())) {
            // If there is enough data and the proposal is complete (the response has the "summary" key) then we move
            // to ProposalReady for a dry run or to the Rebalancing state for a full run
            KafkaRebalanceState ready = dryrun ? KafkaRebalanceState.ProposalReady : KafkaRebalanceState.Rebalancing;
            return buildRebalanceStatus(kafkaRebalance, response.getUserTaskId(), ready, response.getJson(), validate(reconciliation, kafkaRebalance));
        } else {
            throw new CruiseControlRestException("Rebalance returned unknown response: " + response.toString());
        }
    });
}

Example 4 with KafkaRebalanceState

use of io.strimzi.api.kafka.model.balancing.KafkaRebalanceState in project strimzi by strimzi.

the class KafkaRebalanceAssemblyOperator method reconcile.

private Future<Void> reconcile(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, KafkaRebalanceState currentState, KafkaRebalanceAnnotation rebalanceAnnotation) {
    LOGGER.infoCr(reconciliation, "Rebalance action from state [{}]", currentState);
    if (Annotations.isReconciliationPausedWithAnnotation(kafkaRebalance)) {
        // we need to do this check again because it was triggered by a watcher
        KafkaRebalanceStatus status = new KafkaRebalanceStatus();
        Set<Condition> unknownAndDeprecatedConditions = validate(reconciliation, kafkaRebalance);
        unknownAndDeprecatedConditions.add(StatusUtils.getPausedCondition());
        status.setConditions(new ArrayList<>(unknownAndDeprecatedConditions));
        return updateStatus(reconciliation, kafkaRebalance, status, null).compose(i -> Future.succeededFuture());
    }
    RebalanceOptions.RebalanceOptionsBuilder rebalanceOptionsBuilder = convertRebalanceSpecToRebalanceOptions(kafkaRebalance.getSpec(), usingJbodStorage);
    return computeNextStatus(reconciliation, host, apiClient, kafkaRebalance, currentState, rebalanceAnnotation, rebalanceOptionsBuilder).compose(desiredStatusAndMap -> {
        // do a new get to retrieve the current resource state.
        return kafkaRebalanceOperator.getAsync(reconciliation.namespace(), reconciliation.name()).compose(currentKafkaRebalance -> {
            if (currentKafkaRebalance != null) {
                return configMapOperator.reconcile(reconciliation, kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName(), desiredStatusAndMap.getLoadMap()).compose(i -> updateStatus(reconciliation, currentKafkaRebalance, desiredStatusAndMap.getStatus(), null)).compose(updatedKafkaRebalance -> {
                    LOGGER.infoCr(reconciliation, "State updated to [{}] with annotation {}={} ", rebalanceStateConditionType(updatedKafkaRebalance.getStatus()), ANNO_STRIMZI_IO_REBALANCE, rawRebalanceAnnotation(updatedKafkaRebalance));
                    if (hasRebalanceAnnotation(updatedKafkaRebalance)) {
                        LOGGER.debugCr(reconciliation, "Removing annotation {}={}", ANNO_STRIMZI_IO_REBALANCE, rawRebalanceAnnotation(updatedKafkaRebalance));
                        // Updated KafkaRebalance has rebalance annotation removed as
                        // action specified by user has been completed.
                        KafkaRebalance patchedKafkaRebalance = new KafkaRebalanceBuilder(updatedKafkaRebalance).editMetadata().removeFromAnnotations(ANNO_STRIMZI_IO_REBALANCE).endMetadata().build();
                        return kafkaRebalanceOperator.patchAsync(reconciliation, patchedKafkaRebalance);
                    } else {
                        LOGGER.debugCr(reconciliation, "No annotation {}", ANNO_STRIMZI_IO_REBALANCE);
                        return Future.succeededFuture();
                    }
                }).mapEmpty();
            } else {
                return Future.succeededFuture();
            }
        }, exception -> {
            LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", exception.getMessage());
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty();
        });
    }, exception -> {
        LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", exception.getMessage());
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty();
    });
}

Example 5 with KafkaRebalanceState

use of io.strimzi.api.kafka.model.balancing.KafkaRebalanceState in project strimzi-kafka-operator by strimzi.

the class KafkaRebalanceAssemblyOperator method reconcile.

private Future<Void> reconcile(Reconciliation reconciliation, String host, CruiseControlApi apiClient, KafkaRebalance kafkaRebalance, KafkaRebalanceState currentState, KafkaRebalanceAnnotation rebalanceAnnotation) {
    LOGGER.infoCr(reconciliation, "Rebalance action from state [{}]", currentState);
    if (Annotations.isReconciliationPausedWithAnnotation(kafkaRebalance)) {
        // we need to do this check again because it was triggered by a watcher
        KafkaRebalanceStatus status = new KafkaRebalanceStatus();
        Set<Condition> unknownAndDeprecatedConditions = validate(reconciliation, kafkaRebalance);
        unknownAndDeprecatedConditions.add(StatusUtils.getPausedCondition());
        status.setConditions(new ArrayList<>(unknownAndDeprecatedConditions));
        return updateStatus(reconciliation, kafkaRebalance, status, null).compose(i -> Future.succeededFuture());
    }
    RebalanceOptions.RebalanceOptionsBuilder rebalanceOptionsBuilder = convertRebalanceSpecToRebalanceOptions(kafkaRebalance.getSpec(), usingJbodStorage);
    return computeNextStatus(reconciliation, host, apiClient, kafkaRebalance, currentState, rebalanceAnnotation, rebalanceOptionsBuilder).compose(desiredStatusAndMap -> {
        // do a new get to retrieve the current resource state.
        return kafkaRebalanceOperator.getAsync(reconciliation.namespace(), reconciliation.name()).compose(currentKafkaRebalance -> {
            if (currentKafkaRebalance != null) {
                return configMapOperator.reconcile(reconciliation, kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName(), desiredStatusAndMap.getLoadMap()).compose(i -> updateStatus(reconciliation, currentKafkaRebalance, desiredStatusAndMap.getStatus(), null)).compose(updatedKafkaRebalance -> {
                    LOGGER.infoCr(reconciliation, "State updated to [{}] with annotation {}={} ", rebalanceStateConditionType(updatedKafkaRebalance.getStatus()), ANNO_STRIMZI_IO_REBALANCE, rawRebalanceAnnotation(updatedKafkaRebalance));
                    if (hasRebalanceAnnotation(updatedKafkaRebalance)) {
                        LOGGER.debugCr(reconciliation, "Removing annotation {}={}", ANNO_STRIMZI_IO_REBALANCE, rawRebalanceAnnotation(updatedKafkaRebalance));
                        // Updated KafkaRebalance has rebalance annotation removed as
                        // action specified by user has been completed.
                        KafkaRebalance patchedKafkaRebalance = new KafkaRebalanceBuilder(updatedKafkaRebalance).editMetadata().removeFromAnnotations(ANNO_STRIMZI_IO_REBALANCE).endMetadata().build();
                        return kafkaRebalanceOperator.patchAsync(reconciliation, patchedKafkaRebalance);
                    } else {
                        LOGGER.debugCr(reconciliation, "No annotation {}", ANNO_STRIMZI_IO_REBALANCE);
                        return Future.succeededFuture();
                    }
                }).mapEmpty();
            } else {
                return Future.succeededFuture();
            }
        }, exception -> {
            LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", exception.getMessage());
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty();
        });
    }, exception -> {
        LOGGER.errorCr(reconciliation, "Status updated to [NotReady] due to error: {}", exception.getMessage());
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty();
    });
}