Examples with JbodStorage io.strimzi.api.kafka.model.storage.JbodStorage used on opensource projects

Example 26 with JbodStorage

use of io.strimzi.api.kafka.model.storage.JbodStorage in project strimzi-kafka-operator by strimzi.

the class VolumeUtils method createPodSetVolumes.

/**
 * Generates the list of data volumes as used in PodSets and individual Pods. This includes both ephemeral and
 * persistent data volumes. This method calls itself recursively to create the volumes from a JBOD storage array.
 * When it does so, it sets the {@code jbod} parameter to {@code true}. When called from outside, it should be set
 * to {@code false}.
 *
 * @param podName   Name of the pod used to name the volumes
 * @param storage   Storage configuration
 * @param jbod      Indicates that the storage is part of JBOD storage and volume names are created accordingly
 *
 * @return          List of data volumes to be included in the StrimziPodSet pod
 */
public static List<Volume> createPodSetVolumes(String podName, Storage storage, boolean jbod) {
    List<Volume> volumes = new ArrayList<>();
    if (storage != null) {
        if (storage instanceof JbodStorage) {
            for (SingleVolumeStorage volume : ((JbodStorage) storage).getVolumes()) {
                // it's called recursively for setting the information from the current volume
                volumes.addAll(createPodSetVolumes(podName, volume, true));
            }
        } else if (storage instanceof EphemeralStorage) {
            EphemeralStorage ephemeralStorage = (EphemeralStorage) storage;
            volumes.add(createEmptyDirVolume(createVolumePrefix(ephemeralStorage.getId(), jbod), ephemeralStorage.getSizeLimit(), null));
        } else if (storage instanceof PersistentClaimStorage) {
            String name = createVolumePrefix(((PersistentClaimStorage) storage).getId(), jbod);
            volumes.add(createPvcVolume(name, name + "-" + podName));
        }
    }
    return volumes;
}

Example 27 with JbodStorage

use of io.strimzi.api.kafka.model.storage.JbodStorage in project strimzi-kafka-operator by strimzi.

the class KafkaRebalanceAssemblyOperator method reconcileRebalance.

/**
 * Reconcile loop for the KafkaRebalance
 */
/* test */
Future<Void> reconcileRebalance(Reconciliation reconciliation, KafkaRebalance kafkaRebalance) {
    if (kafkaRebalance == null) {
        LOGGER.infoCr(reconciliation, "Rebalance resource deleted");
        return Future.succeededFuture();
    }
    String clusterName = kafkaRebalance.getMetadata().getLabels() == null ? null : kafkaRebalance.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL);
    String clusterNamespace = kafkaRebalance.getMetadata().getNamespace();
    if (clusterName == null) {
        LOGGER.warnCr(reconciliation, "Resource lacks label '{}': No cluster related to a possible rebalance.", Labels.STRIMZI_CLUSTER_LABEL);
        return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException("Resource lacks label '" + Labels.STRIMZI_CLUSTER_LABEL + "': No cluster related to a possible rebalance.")).mapEmpty();
    }
    // Get associated Kafka cluster state
    return kafkaOperator.getAsync(clusterNamespace, clusterName).compose(kafka -> {
        if (kafka == null) {
            LOGGER.warnCr(reconciliation, "Kafka resource '{}' identified by label '{}' does not exist in namespace {}.", clusterName, Labels.STRIMZI_CLUSTER_LABEL, clusterNamespace);
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new NoSuchResourceException("Kafka resource '" + clusterName + "' identified by label '" + Labels.STRIMZI_CLUSTER_LABEL + "' does not exist in namespace " + clusterNamespace + ".")).mapEmpty();
        } else if (!Util.matchesSelector(kafkaSelector, kafka)) {
            LOGGER.debugCr(reconciliation, "{} {} in namespace {} belongs to a Kafka cluster {} which does not match label selector {} and will be ignored", kind(), kafkaRebalance.getMetadata().getName(), clusterNamespace, clusterName, kafkaSelector.get().getMatchLabels());
            return Future.succeededFuture();
        } else if (kafka.getSpec().getCruiseControl() == null) {
            LOGGER.warnCr(reconciliation, "Kafka resource lacks 'cruiseControl' declaration : No deployed Cruise Control for doing a rebalance.");
            return updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), new InvalidResourceException("Kafka resource lacks 'cruiseControl' declaration " + ": No deployed Cruise Control for doing a rebalance.")).mapEmpty();
        }
        if (kafka.getSpec().getKafka().getStorage() instanceof JbodStorage) {
            usingJbodStorage = true;
        }
        String ccSecretName = CruiseControlResources.secretName(clusterName);
        String ccApiSecretName = CruiseControlResources.apiSecretName(clusterName);
        Future<Secret> ccSecretFuture = secretOperations.getAsync(clusterNamespace, ccSecretName);
        Future<Secret> ccApiSecretFuture = secretOperations.getAsync(clusterNamespace, ccApiSecretName);
        return CompositeFuture.join(ccSecretFuture, ccApiSecretFuture).compose(compositeFuture -> {
            Secret ccSecret = compositeFuture.resultAt(0);
            if (ccSecret == null) {
                return Future.failedFuture(Util.missingSecretException(clusterNamespace, ccSecretName));
            }
            Secret ccApiSecret = compositeFuture.resultAt(1);
            if (ccApiSecret == null) {
                return Future.failedFuture(Util.missingSecretException(clusterNamespace, ccApiSecretName));
            }
            CruiseControlConfiguration c = new CruiseControlConfiguration(reconciliation, kafka.getSpec().getCruiseControl().getConfig().entrySet());
            boolean apiAuthEnabled = CruiseControl.isApiAuthEnabled(c);
            boolean apiSslEnabled = CruiseControl.isApiSslEnabled(c);
            CruiseControlApi apiClient = cruiseControlClientProvider(ccSecret, ccApiSecret, apiAuthEnabled, apiSslEnabled);
            // get latest KafkaRebalance state as it may have changed
            return kafkaRebalanceOperator.getAsync(kafkaRebalance.getMetadata().getNamespace(), kafkaRebalance.getMetadata().getName()).compose(currentKafkaRebalance -> {
                KafkaRebalanceStatus kafkaRebalanceStatus = currentKafkaRebalance.getStatus();
                KafkaRebalanceState currentState;
                // cluster rebalance is new or it is in one of the others states
                if (kafkaRebalanceStatus == null || kafkaRebalanceStatus.getConditions().stream().filter(cond -> "ReconciliationPaused".equals(cond.getType())).findAny().isPresent()) {
                    currentState = KafkaRebalanceState.New;
                } else {
                    String rebalanceStateType = rebalanceStateConditionType(kafkaRebalanceStatus);
                    if (rebalanceStateType == null) {
                        throw new RuntimeException("Unable to find KafkaRebalance State in current KafkaRebalance status");
                    }
                    currentState = KafkaRebalanceState.valueOf(rebalanceStateType);
                }
                // Check annotation
                KafkaRebalanceAnnotation rebalanceAnnotation = rebalanceAnnotation(reconciliation, currentKafkaRebalance);
                return reconcile(reconciliation, cruiseControlHost(clusterName, clusterNamespace), apiClient, currentKafkaRebalance, currentState, rebalanceAnnotation).mapEmpty();
            }, exception -> Future.failedFuture(exception).mapEmpty());
        });
    }, exception -> updateStatus(reconciliation, kafkaRebalance, new KafkaRebalanceStatus(), exception).mapEmpty());
}

Example 28 with JbodStorage

use of io.strimzi.api.kafka.model.storage.JbodStorage in project strimzi-kafka-operator by strimzi.

the class AlternativeReconcileTriggersST method testAddingAndRemovingJbodVolumes.

/**
 * Adding and removing JBOD volumes requires rolling updates in the sequential order. Otherwise the StatefulSet does
 * not like it. This tests tries to add and remove volume from JBOD to test both of these situations.
 */
@ParallelNamespaceTest
void testAddingAndRemovingJbodVolumes(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    PersistentClaimStorage vol0 = new PersistentClaimStorageBuilder().withId(0).withSize("1Gi").withDeleteClaim(true).build();
    PersistentClaimStorage vol1 = new PersistentClaimStorageBuilder().withId(1).withSize("1Gi").withDeleteClaim(true).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(clusterName, 3, 3, new JbodStorageBuilder().addToVolumes(vol0).build()).build());
    final String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // Add Jbod volume to Kafka => triggers RU
    LOGGER.info("Add JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().add(vol1);
    }, namespaceName);
    // Wait util it rolls
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // Remove Jbod volume to Kafka => triggers RU
    LOGGER.info("Remove JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().remove(vol1);
    }, namespaceName);
    // Wait util it rolls
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}