Examples with PersistentClaimStorage io.strimzi.api.kafka.model.storage.PersistentClaimStorage used on opensource projects

Example 26 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class AlternativeReconcileTriggersST method testAddingAndRemovingJbodVolumes.

/**
 * Adding and removing JBOD volumes requires rolling updates in the sequential order. Otherwise the StatefulSet does
 * not like it. This tests tries to add and remove volume from JBOD to test both of these situations.
 */
@ParallelNamespaceTest
void testAddingAndRemovingJbodVolumes(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    PersistentClaimStorage vol0 = new PersistentClaimStorageBuilder().withId(0).withSize("1Gi").withDeleteClaim(true).build();
    PersistentClaimStorage vol1 = new PersistentClaimStorageBuilder().withId(1).withSize("1Gi").withDeleteClaim(true).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(clusterName, 3, 3, new JbodStorageBuilder().addToVolumes(vol0).build()).build());
    final String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // Add Jbod volume to Kafka => triggers RU
    LOGGER.info("Add JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().add(vol1);
    }, namespaceName);
    // Wait util it rolls
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // Remove Jbod volume to Kafka => triggers RU
    LOGGER.info("Remove JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().remove(vol1);
    }, namespaceName);
    // Wait util it rolls
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}

Example 27 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class JbodStorageTest method testJbodStorageCreatesPersistentVolumeClaimsMatchingKafkaVolumes.

@Test
public void testJbodStorageCreatesPersistentVolumeClaimsMatchingKafkaVolumes(VertxTestContext context) {
    Checkpoint async = context.checkpoint();
    operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME)).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        for (int i = 0; i < this.kafka.getSpec().getKafka().getReplicas(); i++) {
            int podId = i;
            for (SingleVolumeStorage volume : this.volumes) {
                if (volume instanceof PersistentClaimStorage) {
                    String expectedPvcName = VolumeUtils.createVolumePrefix(volume.getId(), true) + "-" + KafkaCluster.kafkaPodName(NAME, podId);
                    List<PersistentVolumeClaim> matchingPvcs = pvcs.stream().filter(pvc -> pvc.getMetadata().getName().equals(expectedPvcName)).collect(Collectors.toList());
                    assertThat("Exactly one pvc should have the name " + expectedPvcName + " in :\n" + pvcs.toString(), matchingPvcs, Matchers.hasSize(1));
                    PersistentVolumeClaim pvc = matchingPvcs.get(0);
                    boolean isDeleteClaim = ((PersistentClaimStorage) volume).isDeleteClaim();
                    assertThat("deleteClaim value did not match for volume : " + volume.toString(), Annotations.booleanAnnotation(pvc, AbstractModel.ANNO_STRIMZI_IO_DELETE_CLAIM, false), is(isDeleteClaim));
                }
            }
        }
        async.flag();
    })));
}

Example 28 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class KafkaAssemblyOperatorMockTest method testReconcileUpdatesKafkaWithChangedDeleteClaim.

/**
 * Test that we can change the deleteClaim flag, and that it's honoured
 */
@ParameterizedTest
@MethodSource("data")
public void testReconcileUpdatesKafkaWithChangedDeleteClaim(Params params, VertxTestContext context) {
    init(params);
    assumeTrue(kafkaStorage instanceof PersistentClaimStorage, "Kafka delete claims do not apply to non-persistent volumes");
    Map<String, String> kafkaLabels = new HashMap<>();
    kafkaLabels.put(Labels.STRIMZI_KIND_LABEL, Kafka.RESOURCE_KIND);
    kafkaLabels.put(Labels.STRIMZI_CLUSTER_LABEL, CLUSTER_NAME);
    kafkaLabels.put(Labels.STRIMZI_NAME_LABEL, KafkaCluster.kafkaClusterName(CLUSTER_NAME));
    Map<String, String> zkLabels = new HashMap<>();
    zkLabels.put(Labels.STRIMZI_KIND_LABEL, Kafka.RESOURCE_KIND);
    zkLabels.put(Labels.STRIMZI_CLUSTER_LABEL, CLUSTER_NAME);
    zkLabels.put(Labels.STRIMZI_NAME_LABEL, ZookeeperCluster.zookeeperClusterName(CLUSTER_NAME));
    AtomicReference<Set<String>> kafkaPvcs = new AtomicReference<>();
    AtomicReference<Set<String>> zkPvcs = new AtomicReference<>();
    AtomicBoolean originalKafkaDeleteClaim = new AtomicBoolean();
    Checkpoint async = context.checkpoint();
    initialReconcile(context).onComplete(context.succeeding(v -> context.verify(() -> {
        kafkaPvcs.set(client.persistentVolumeClaims().inNamespace(NAMESPACE).withLabels(kafkaLabels).list().getItems().stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet()));
        zkPvcs.set(client.persistentVolumeClaims().inNamespace(NAMESPACE).withLabels(zkLabels).list().getItems().stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet()));
        originalKafkaDeleteClaim.set(deleteClaim(kafkaStorage));
        // Try to update the storage class
        Kafka updatedStorageKafka = new KafkaBuilder(cluster).editSpec().editKafka().withNewPersistentClaimStorage().withSize("123").withStorageClass("foo").withDeleteClaim(!originalKafkaDeleteClaim.get()).endPersistentClaimStorage().endKafka().endSpec().build();
        kafkaAssembly(NAMESPACE, CLUSTER_NAME).patch(updatedStorageKafka);
        LOGGER.info("Updating with changed delete claim");
    }))).compose(v -> operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME))).onComplete(context.succeeding(v -> context.verify(() -> {
        // check that the new delete-claim annotation is on the PVCs
        for (String pvcName : kafkaPvcs.get()) {
            assertThat(client.persistentVolumeClaims().inNamespace(NAMESPACE).withName(pvcName).get().getMetadata().getAnnotations(), hasEntry(AbstractModel.ANNO_STRIMZI_IO_DELETE_CLAIM, String.valueOf(!originalKafkaDeleteClaim.get())));
        }
        kafkaAssembly(NAMESPACE, CLUSTER_NAME).withPropagationPolicy(DeletionPropagation.FOREGROUND).delete();
        LOGGER.info("Reconciling again -> delete");
    }))).compose(v -> operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME))).onComplete(context.succeeding(v -> async.flag()));
}

Example 29 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaCluster method handleExistingVolume.

private <V extends SingleVolumeStorage> void handleExistingVolume(V v, PersistentClaimStorageBuilder builder) {
    if (v instanceof PersistentClaimStorage) {
        PersistentClaimStorage persistentClaimStorage = (PersistentClaimStorage) v;
        if (persistentClaimStorage.getOverrides() != null && !persistentClaimStorage.getOverrides().isEmpty()) {
            log.trace("Reusing storage overrides on existing Kafka");
            builder.withOverrides(persistentClaimStorage.getOverrides());
        } else {
            log.trace("Setting default StorageClass on Kafka");
            builder.withStorageClass(config.getKafka().getStorageClass());
        }
    } else {
        log.error("Existing Volume is not an instance of PersistentClaimStorage. This shouldn't happen.");
    }
}

Example 30 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaClusterTest method testExistingStorageClassOverridesDontGetUpdated.

@Test
void testExistingStorageClassOverridesDontGetUpdated() {
    ManagedKafka mk = exampleManagedKafka("60Gi");
    Kafka defaultKafka = kafkaCluster.kafkaFrom(mk, null);
    JbodStorage defaultKafkaStorage = (JbodStorage) defaultKafka.getSpec().getKafka().getStorage();
    PersistentClaimStorage defaultZookeeperStorage = (PersistentClaimStorage) defaultKafka.getSpec().getZookeeper().getStorage();
    JbodStorage kafkaStorageWithOverrides = new JbodStorageBuilder(defaultKafkaStorage).withVolumes(defaultKafkaStorage.getVolumes().stream().map(v -> {
        PersistentClaimStorage pcs = (PersistentClaimStorage) v;
        pcs.setStorageClass(null);
        pcs.setOverrides(buildStorageOverrides());
        return pcs;
    }).collect(Collectors.toList())).build();
    PersistentClaimStorage zookeeperStorageWithOverrides = new PersistentClaimStorageBuilder(defaultZookeeperStorage).withStorageClass(null).withOverrides(buildStorageOverrides()).build();
    Kafka kafkaWithOverrides = new KafkaBuilder(defaultKafka).editSpec().editKafka().withStorage(kafkaStorageWithOverrides).endKafka().editZookeeper().withStorage(zookeeperStorageWithOverrides).endZookeeper().endSpec().build();
    Kafka reconciledKafka = kafkaCluster.kafkaFrom(mk, kafkaWithOverrides);
    assertNull(((PersistentClaimStorage) reconciledKafka.getSpec().getZookeeper().getStorage()).getStorageClass());
    assertEquals(buildStorageOverrides(), ((PersistentClaimStorage) reconciledKafka.getSpec().getZookeeper().getStorage()).getOverrides());
    ((JbodStorage) reconciledKafka.getSpec().getKafka().getStorage()).getVolumes().stream().forEach(v -> {
        assertNull(((PersistentClaimStorage) v).getStorageClass());
        assertEquals(buildStorageOverrides(), ((PersistentClaimStorage) v).getOverrides());
    });
}