Examples with PersistentClaimStorageBuilder io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder used on opensource projects

Example 91 with PersistentClaimStorageBuilder

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder in project strimzi-kafka-operator by strimzi.

the class KafkaST method testPersistentStorageSize.

@ParallelNamespaceTest
@Tag(INTERNAL_CLIENTS_USED)
@KRaftNotSupported("JBOD is not supported by KRaft mode and is used in this test case.")
void testPersistentStorageSize(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext);
    final String[] diskSizes = { "70Gi", "20Gi" };
    final int kafkaRepl = 2;
    final int diskCount = 2;
    JbodStorage jbodStorage = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withDeleteClaim(false).withId(0).withSize(diskSizes[0]).build(), new PersistentClaimStorageBuilder().withDeleteClaim(false).withId(1).withSize(diskSizes[1]).build()).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(testStorage.getClusterName(), kafkaRepl).editSpec().editKafka().withStorage(jbodStorage).endKafka().editZookeeper().withReplicas(1).endZookeeper().endSpec().build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(testStorage.getClusterName(), testStorage.getTopicName()).build());
    List<PersistentVolumeClaim> volumes = kubeClient(testStorage.getNamespaceName()).listPersistentVolumeClaims(testStorage.getNamespaceName(), testStorage.getClusterName()).stream().filter(persistentVolumeClaim -> persistentVolumeClaim.getMetadata().getName().contains(testStorage.getClusterName())).collect(Collectors.toList());
    checkStorageSizeForVolumes(volumes, diskSizes, kafkaRepl, diskCount);
    KafkaClients kafkaClients = new KafkaClientsBuilder().withTopicName(testStorage.getTopicName()).withBootstrapAddress(KafkaResources.plainBootstrapAddress(testStorage.getClusterName())).withNamespaceName(testStorage.getNamespaceName()).withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withMessageCount(MESSAGE_COUNT).build();
    resourceManager.createResource(extensionContext, kafkaClients.producerStrimzi(), kafkaClients.consumerStrimzi());
    ClientUtils.waitForClientsSuccess(testStorage.getProducerName(), testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
}

Example 92 with PersistentClaimStorageBuilder

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder in project strimzi-kafka-operator by strimzi.

the class KafkaST method testKafkaJBODDeleteClaimsFalse.

@ParallelNamespaceTest
@KRaftNotSupported("JBOD is not supported by KRaft mode and is used in this test case.")
void testKafkaJBODDeleteClaimsFalse(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final int kafkaReplicas = 2;
    final String diskSizeGi = "10";
    JbodStorage jbodStorage = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withDeleteClaim(false).withId(0).withSize(diskSizeGi + "Gi").build(), new PersistentClaimStorageBuilder().withDeleteClaim(false).withId(1).withSize(diskSizeGi + "Gi").build()).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(clusterName, kafkaReplicas, jbodStorage).build());
    // kafka cluster already deployed
    verifyVolumeNamesAndLabels(namespaceName, clusterName, kafkaReplicas, 2, diskSizeGi);
    int volumesCount = kubeClient(namespaceName).listPersistentVolumeClaims(namespaceName, clusterName).size();
    LOGGER.info("Deleting cluster");
    cmdKubeClient(namespaceName).deleteByName("kafka", clusterName);
    LOGGER.info("Waiting for PVC deletion");
    PersistentVolumeClaimUtils.waitForJbodStorageDeletion(namespaceName, volumesCount, jbodStorage, clusterName);
}

Example 93 with PersistentClaimStorageBuilder

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder in project strimzi-kafka-operator by strimzi.

the class KafkaAssemblyOperatorManualRollingUpdatesTest method manualPodCleanup.

public void manualPodCleanup(VertxTestContext context, boolean useStrimziPodSets) {
    Kafka kafka = new KafkaBuilder().withNewMetadata().withName(clusterName).withNamespace(namespace).withGeneration(2L).endMetadata().withNewSpec().withNewKafka().withReplicas(3).withListeners(new GenericKafkaListenerBuilder().withName("plain").withPort(9092).withType(KafkaListenerType.INTERNAL).withTls(false).build()).withNewJbodStorage().withVolumes(new PersistentClaimStorageBuilder().withId(0).withSize("100Gi").build(), new PersistentClaimStorageBuilder().withId(1).withSize("100Gi").build()).endJbodStorage().endKafka().withNewZookeeper().withReplicas(3).withNewPersistentClaimStorage().withSize("100Gi").endPersistentClaimStorage().endZookeeper().endSpec().build();
    KafkaCluster kafkaCluster = KafkaCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kafka, VERSIONS);
    ZookeeperCluster zkCluster = ZookeeperCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kafka, VERSIONS);
    ResourceOperatorSupplier supplier = ResourceUtils.supplierWithMocks(false);
    if (useStrimziPodSets) {
        CrdOperator<KubernetesClient, StrimziPodSet, StrimziPodSetList> mockPodSetOps = supplier.strimziPodSetOperator;
        when(mockPodSetOps.getAsync(any(), eq(zkCluster.getName()))).thenReturn(Future.succeededFuture(zkCluster.generatePodSet(kafka.getSpec().getZookeeper().getReplicas(), false, null, null, null)));
        when(mockPodSetOps.deleteAsync(any(), any(), eq(zkCluster.getName()), anyBoolean())).thenReturn(Future.succeededFuture());
        when(mockPodSetOps.reconcile(any(), any(), eq(zkCluster.getName()), any())).thenReturn(Future.succeededFuture());
        when(mockPodSetOps.getAsync(any(), eq(kafkaCluster.getName()))).thenReturn(Future.succeededFuture(kafkaCluster.generatePodSet(kafka.getSpec().getKafka().getReplicas(), false, null, null, null)));
        when(mockPodSetOps.deleteAsync(any(), any(), eq(kafkaCluster.getName()), anyBoolean())).thenReturn(Future.succeededFuture());
        when(mockPodSetOps.reconcile(any(), any(), eq(kafkaCluster.getName()), any())).thenReturn(Future.succeededFuture());
        StatefulSetOperator mockStsOps = supplier.stsOperations;
        when(mockStsOps.getAsync(any(), any())).thenReturn(Future.succeededFuture(null));
    } else {
        StatefulSetOperator mockStsOps = supplier.stsOperations;
        when(mockStsOps.getAsync(any(), eq(zkCluster.getName()))).thenReturn(Future.succeededFuture(zkCluster.generateStatefulSet(false, null, null)));
        when(mockStsOps.getAsync(any(), eq(kafkaCluster.getName()))).thenReturn(Future.succeededFuture(kafkaCluster.generateStatefulSet(false, null, null, null)));
        when(mockStsOps.deleteAsync(any(), any(), eq(zkCluster.getName()), anyBoolean())).thenReturn(Future.succeededFuture());
        when(mockStsOps.deleteAsync(any(), any(), eq(kafkaCluster.getName()), anyBoolean())).thenReturn(Future.succeededFuture());
        when(mockStsOps.reconcile(any(), any(), eq(zkCluster.getName()), any())).thenReturn(Future.succeededFuture());
        when(mockStsOps.reconcile(any(), any(), eq(kafkaCluster.getName()), any())).thenReturn(Future.succeededFuture());
        CrdOperator<KubernetesClient, StrimziPodSet, StrimziPodSetList> mockPodSetOps = supplier.strimziPodSetOperator;
        when(mockPodSetOps.getAsync(any(), any())).thenReturn(Future.succeededFuture(null));
    }
    PodOperator mockPodOps = supplier.podOperations;
    when(mockPodOps.listAsync(any(), eq(zkCluster.getSelectorLabels()))).thenAnswer(i -> {
        List<Pod> pods = new ArrayList<>();
        pods.add(podWithName("my-cluster-zookeeper-0"));
        pods.add(podWithNameAndAnnotations("my-cluster-zookeeper-1", Collections.singletonMap(AbstractScalableResourceOperator.ANNO_STRIMZI_IO_DELETE_POD_AND_PVC, "true")));
        pods.add(podWithName("my-cluster-zookeeper-2"));
        return Future.succeededFuture(pods);
    });
    when(mockPodOps.listAsync(any(), eq(kafkaCluster.getSelectorLabels()))).thenAnswer(i -> {
        List<Pod> pods = new ArrayList<>();
        pods.add(podWithNameAndAnnotations("my-cluster-kafka-0", Collections.singletonMap(AbstractScalableResourceOperator.ANNO_STRIMZI_IO_DELETE_POD_AND_PVC, "true")));
        pods.add(podWithName("my-cluster-kafka-1"));
        pods.add(podWithName("my-cluster-kafka-2"));
        return Future.succeededFuture(pods);
    });
    when(mockPodOps.readiness(any(), any(), any(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    PvcOperator mockPvcOps = supplier.pvcOperations;
    when(mockPvcOps.listAsync(any(), eq(zkCluster.getSelectorLabels()))).thenReturn(Future.succeededFuture(zkCluster.generatePersistentVolumeClaims()));
    when(mockPvcOps.listAsync(any(), eq(kafkaCluster.getSelectorLabels()))).thenReturn(Future.succeededFuture(kafkaCluster.generatePersistentVolumeClaims(kafkaCluster.getStorage())));
    CrdOperator mockKafkaOps = supplier.kafkaOperator;
    when(mockKafkaOps.getAsync(eq(namespace), eq(clusterName))).thenReturn(Future.succeededFuture(kafka));
    when(mockKafkaOps.get(eq(namespace), eq(clusterName))).thenReturn(kafka);
    when(mockKafkaOps.updateStatusAsync(any(), any())).thenReturn(Future.succeededFuture());
    ClusterOperatorConfig config;
    if (useStrimziPodSets) {
        config = ResourceUtils.dummyClusterOperatorConfig(VERSIONS, ClusterOperatorConfig.DEFAULT_OPERATION_TIMEOUT_MS, "+UseStrimziPodSets");
    } else {
        config = ResourceUtils.dummyClusterOperatorConfig(VERSIONS, ClusterOperatorConfig.DEFAULT_OPERATION_TIMEOUT_MS);
    }
    MockKafkaAssemblyOperator kao = new MockKafkaAssemblyOperator(vertx, new PlatformFeaturesAvailability(false, kubernetesVersion), certManager, passwordGenerator, supplier, config);
    Checkpoint async = context.checkpoint();
    kao.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, namespace, clusterName)).onComplete(context.succeeding(v -> context.verify(() -> {
        // Verify Zookeeper rolling updates
        assertThat(kao.cleanedPodsCount, is(2));
        assertThat(kao.cleanedPods, Matchers.containsInAnyOrder("my-cluster-zookeeper-1", "my-cluster-kafka-0"));
        assertThat(kao.cleanedPvcs, Matchers.containsInAnyOrder("data-my-cluster-zookeeper-1", "data-0-my-cluster-kafka-0", "data-1-my-cluster-kafka-0"));
        assertThat(kao.createdPvcs, Matchers.containsInAnyOrder("data-my-cluster-zookeeper-1", "data-0-my-cluster-kafka-0", "data-1-my-cluster-kafka-0"));
        async.flag();
    })));
}

Example 94 with PersistentClaimStorageBuilder

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaCluster method buildKafkaStorage.

private Storage buildKafkaStorage(ManagedKafka managedKafka, Kafka current) {
    PersistentClaimStorageBuilder builder = new PersistentClaimStorageBuilder().withId(JBOD_VOLUME_ID).withSize(getAdjustedMaxDataRetentionSize(managedKafka, current).getAmount()).withDeleteClaim(DELETE_CLAIM);
    Optional.ofNullable(current).map(Kafka::getSpec).map(KafkaSpec::getKafka).map(KafkaClusterSpec::getStorage).map(this::getExistingVolumesFromJbodStorage).ifPresentOrElse(existingVolumes -> existingVolumes.stream().forEach(v -> handleExistingVolume(v, builder)), () -> builder.withStorageClass(config.getKafka().getStorageClass()));
    return new JbodStorageBuilder().withVolumes(builder.build()).build();
}

Example 95 with PersistentClaimStorageBuilder

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaClusterTest method testExistingStorageClassOverridesDontGetUpdated.

@Test
void testExistingStorageClassOverridesDontGetUpdated() {
    ManagedKafka mk = exampleManagedKafka("60Gi");
    Kafka defaultKafka = kafkaCluster.kafkaFrom(mk, null);
    JbodStorage defaultKafkaStorage = (JbodStorage) defaultKafka.getSpec().getKafka().getStorage();
    PersistentClaimStorage defaultZookeeperStorage = (PersistentClaimStorage) defaultKafka.getSpec().getZookeeper().getStorage();
    JbodStorage kafkaStorageWithOverrides = new JbodStorageBuilder(defaultKafkaStorage).withVolumes(defaultKafkaStorage.getVolumes().stream().map(v -> {
        PersistentClaimStorage pcs = (PersistentClaimStorage) v;
        pcs.setStorageClass(null);
        pcs.setOverrides(buildStorageOverrides());
        return pcs;
    }).collect(Collectors.toList())).build();
    PersistentClaimStorage zookeeperStorageWithOverrides = new PersistentClaimStorageBuilder(defaultZookeeperStorage).withStorageClass(null).withOverrides(buildStorageOverrides()).build();
    Kafka kafkaWithOverrides = new KafkaBuilder(defaultKafka).editSpec().editKafka().withStorage(kafkaStorageWithOverrides).endKafka().editZookeeper().withStorage(zookeeperStorageWithOverrides).endZookeeper().endSpec().build();
    Kafka reconciledKafka = kafkaCluster.kafkaFrom(mk, kafkaWithOverrides);
    assertNull(((PersistentClaimStorage) reconciledKafka.getSpec().getZookeeper().getStorage()).getStorageClass());
    assertEquals(buildStorageOverrides(), ((PersistentClaimStorage) reconciledKafka.getSpec().getZookeeper().getStorage()).getOverrides());
    ((JbodStorage) reconciledKafka.getSpec().getKafka().getStorage()).getVolumes().stream().forEach(v -> {
        assertNull(((PersistentClaimStorage) v).getStorageClass());
        assertEquals(buildStorageOverrides(), ((PersistentClaimStorage) v).getOverrides());
    });
}