Examples with JbodStorageBuilder io.strimzi.api.kafka.model.storage.JbodStorageBuilder used on opensource projects

Example 41 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class JbodStorageMockTest method testReconcileWithNewVolumeAddedToJbodStorage.

@Test
public void testReconcileWithNewVolumeAddedToJbodStorage(VertxTestContext context) {
    Checkpoint async = context.checkpoint();
    // Add a new volume to Jbod Storage
    volumes.add(new PersistentClaimStorageBuilder().withId(2).withDeleteClaim(false).withSize("100Gi").build());
    Kafka kafkaWithNewJbodVolume = new KafkaBuilder(kafka).editSpec().editKafka().withStorage(new JbodStorageBuilder().withVolumes(volumes).build()).endKafka().endSpec().build();
    Set<String> expectedPvcs = expectedPvcs(kafka);
    Set<String> expectedPvcsWithNewJbodStorageVolume = expectedPvcs(kafkaWithNewJbodVolume);
    // reconcile for kafka cluster creation
    operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME)).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        Set<String> pvcsNames = pvcs.stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet());
        assertThat(pvcsNames, is(expectedPvcs));
    }))).compose(v -> {
        Crds.kafkaOperation(client).inNamespace(NAMESPACE).withName(NAME).patch(kafkaWithNewJbodVolume);
        // reconcile kafka cluster with new Jbod storage
        return operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME));
    }).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        Set<String> pvcsNames = pvcs.stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet());
        assertThat(pvcsNames, is(expectedPvcsWithNewJbodStorageVolume));
        async.flag();
    })));
}

Example 42 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class JbodStorageMockTest method testReconcileWithUpdateVolumeIdJbod.

@Test
public void testReconcileWithUpdateVolumeIdJbod(VertxTestContext context) {
    Checkpoint async = context.checkpoint();
    // trying to update id for a volume from in the JBOD storage
    volumes.get(0).setId(3);
    Kafka kafkaWithUpdatedJbodVolume = new KafkaBuilder(this.kafka).editSpec().editKafka().withStorage(new JbodStorageBuilder().withVolumes(volumes).build()).endKafka().endSpec().build();
    Set<String> expectedPvcs = expectedPvcs(kafka);
    Set<String> expectedPvcsWithUpdatedJbodStorageVolume = expectedPvcs(kafkaWithUpdatedJbodVolume);
    // reconcile for kafka cluster creation
    operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME)).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        Set<String> pvcsNames = pvcs.stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet());
        assertThat(pvcsNames, is(expectedPvcs));
    }))).compose(v -> {
        Crds.kafkaOperation(client).inNamespace(NAMESPACE).withName(NAME).patch(kafkaWithUpdatedJbodVolume);
        // reconcile kafka cluster with a Jbod storage volume removed
        return operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME));
    }).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        Set<String> pvcsNames = pvcs.stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet());
        assertThat(pvcsNames, is(expectedPvcsWithUpdatedJbodStorageVolume));
        async.flag();
    })));
}

Example 43 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class JbodStorageMockTest method testReconcileWithVolumeRemovedFromJbodStorage.

@Test
public void testReconcileWithVolumeRemovedFromJbodStorage(VertxTestContext context) {
    Checkpoint async = context.checkpoint();
    // remove a volume from the Jbod Storage
    volumes.remove(0);
    Kafka kafkaWithRemovedJbodVolume = new KafkaBuilder(this.kafka).editSpec().editKafka().withStorage(new JbodStorageBuilder().withVolumes(volumes).build()).endKafka().endSpec().build();
    Set<String> expectedPvcs = expectedPvcs(kafka);
    Set<String> expectedPvcsWithRemovedJbodStorageVolume = expectedPvcs(kafkaWithRemovedJbodVolume);
    // reconcile for kafka cluster creation
    operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME)).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        Set<String> pvcsNames = pvcs.stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet());
        assertThat(pvcsNames, is(expectedPvcs));
    }))).compose(v -> {
        Crds.kafkaOperation(client).inNamespace(NAMESPACE).withName(NAME).patch(kafkaWithRemovedJbodVolume);
        // reconcile kafka cluster with a Jbod storage volume removed
        return operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME));
    }).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        Set<String> pvcsNames = pvcs.stream().map(pvc -> pvc.getMetadata().getName()).collect(Collectors.toSet());
        assertThat(pvcsNames, is(expectedPvcsWithRemovedJbodStorageVolume));
        async.flag();
    })));
}

Example 44 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi-kafka-operator by strimzi.

the class AlternativeReconcileTriggersST method testAddingAndRemovingJbodVolumes.

/**
 * Adding and removing JBOD volumes requires rolling updates in the sequential order. Otherwise the StatefulSet does
 * not like it. This tests tries to add and remove volume from JBOD to test both of these situations.
 */
@ParallelNamespaceTest
@KRaftNotSupported("UserOperator is not supported by KRaft mode and is used in this test case. JBOD is not supported as well.")
void testAddingAndRemovingJbodVolumes(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext, namespace);
    final String continuousTopicName = "continuous-topic";
    final String continuousProducerName = "continuous-" + testStorage.getProducerName();
    final String continuousConsumerName = "continuous-" + testStorage.getConsumerName();
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    PersistentClaimStorage vol0 = new PersistentClaimStorageBuilder().withId(0).withSize("1Gi").withDeleteClaim(true).build();
    PersistentClaimStorage vol1 = new PersistentClaimStorageBuilder().withId(1).withSize("1Gi").withDeleteClaim(true).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(testStorage.getClusterName(), 3, 3, new JbodStorageBuilder().addToVolumes(vol0).build()).build());
    final String kafkaName = KafkaResources.kafkaStatefulSetName(testStorage.getClusterName());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getKafkaSelector());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(testStorage.getClusterName(), testStorage.getTopicName()).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(testStorage.getClusterName(), continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(continuousProducerName).withConsumerName(continuousConsumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(testStorage.getClusterName())).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(testStorage.getNamespaceName()).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi(), kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    resourceManager.createResource(extensionContext, KafkaUserTemplates.tlsUser(testStorage.getClusterName(), testStorage.getUserName()).build());
    KafkaClients clients = new KafkaClientsBuilder().withProducerName(testStorage.getProducerName()).withConsumerName(testStorage.getConsumerName()).withBootstrapAddress(KafkaResources.tlsBootstrapAddress(testStorage.getClusterName())).withTopicName(testStorage.getTopicName()).withMessageCount(MESSAGE_COUNT).withNamespaceName(testStorage.getNamespaceName()).withUserName(testStorage.getUserName()).build();
    resourceManager.createResource(extensionContext, clients.producerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getProducerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // Add Jbod volume to Kafka => triggers RU
    LOGGER.info("Add JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(testStorage.getClusterName(), kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().add(vol1);
    }, testStorage.getNamespaceName());
    // Wait util it rolls
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), 3, kafkaPods);
    // Remove Jbod volume to Kafka => triggers RU
    LOGGER.info("Remove JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(testStorage.getClusterName(), kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().remove(vol1);
    }, testStorage.getNamespaceName());
    // Wait util it rolls
    RollingUpdateUtils.waitTillComponentHasRolled(testStorage.getNamespaceName(), testStorage.getKafkaSelector(), 3, kafkaPods);
    resourceManager.createResource(extensionContext, clients.consumerTlsStrimzi(testStorage.getClusterName()));
    ClientUtils.waitForClientSuccess(testStorage.getConsumerName(), testStorage.getNamespaceName(), MESSAGE_COUNT);
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitForClientsSuccess(continuousProducerName, continuousConsumerName, testStorage.getNamespaceName(), continuousClientsMessageCount);
// ##############################
}

Example 45 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi-kafka-operator by strimzi.

the class KafkaClusterTest method testStorageValidationAfterInitialDeployment.

@ParallelTest
public void testStorageValidationAfterInitialDeployment() {
    assertThrows(InvalidResourceException.class, () -> {
        Storage oldStorage = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withSize("100Gi").build()).build();
        Kafka kafkaAssembly = new KafkaBuilder(ResourceUtils.createKafka(namespace, cluster, replicas, image, healthDelay, healthTimeout, jmxMetricsConfig, configuration, emptyMap())).editSpec().editKafka().withStorage(new JbodStorageBuilder().withVolumes(List.of()).build()).endKafka().endSpec().build();
        KafkaCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kafkaAssembly, VERSIONS, oldStorage, replicas, false);
    });
}