Examples with JbodStorageBuilder io.strimzi.api.kafka.model.storage.JbodStorageBuilder used on opensource projects

Example 26 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class ModelUtilsTest method testStorageSerializationAndDeserialization.

@ParallelTest
public void testStorageSerializationAndDeserialization() {
    Storage jbod = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("100Gi").build(), new PersistentClaimStorageBuilder().withStorageClass("gp2-st1").withDeleteClaim(true).withId(1).withSize("1000Gi").build()).build();
    Storage ephemeral = new EphemeralStorageBuilder().build();
    Storage persistent = new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("100Gi").build();
    assertThat(ModelUtils.decodeStorageFromJson(ModelUtils.encodeStorageToJson(jbod)), is(jbod));
    assertThat(ModelUtils.decodeStorageFromJson(ModelUtils.encodeStorageToJson(ephemeral)), is(ephemeral));
    assertThat(ModelUtils.decodeStorageFromJson(ModelUtils.encodeStorageToJson(persistent)), is(persistent));
}

Example 27 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class AlternativeReconcileTriggersST method testAddingAndRemovingJbodVolumes.

/**
 * Adding and removing JBOD volumes requires rolling updates in the sequential order. Otherwise the StatefulSet does
 * not like it. This tests tries to add and remove volume from JBOD to test both of these situations.
 */
@ParallelNamespaceTest
void testAddingAndRemovingJbodVolumes(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    PersistentClaimStorage vol0 = new PersistentClaimStorageBuilder().withId(0).withSize("1Gi").withDeleteClaim(true).build();
    PersistentClaimStorage vol1 = new PersistentClaimStorageBuilder().withId(1).withSize("1Gi").withDeleteClaim(true).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(clusterName, 3, 3, new JbodStorageBuilder().addToVolumes(vol0).build()).build());
    final String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // Add Jbod volume to Kafka => triggers RU
    LOGGER.info("Add JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().add(vol1);
    }, namespaceName);
    // Wait util it rolls
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // Remove Jbod volume to Kafka => triggers RU
    LOGGER.info("Remove JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().remove(vol1);
    }, namespaceName);
    // Wait util it rolls
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}

Example 28 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class StorageDiffTest method testJbodDiffWithNewVolume.

@ParallelTest
public void testJbodDiffWithNewVolume() {
    Storage jbod = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("1000Gi").build()).build();
    Storage jbod2 = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("1000Gi").build(), new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(true).withId(1).withSize("1000Gi").build()).build();
    Storage jbod3 = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-st1").withDeleteClaim(false).withId(0).withSize("100Gi").build(), new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(true).withId(1).withSize("1000Gi").build()).build();
    Storage jbod4 = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(2).withSize("1000Gi").build()).build();
    Storage jbod5 = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(true).withId(1).withSize("1000Gi").build()).build();
    Storage jbod6 = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("1000Gi").build(), new PersistentClaimStorageBuilder().withStorageClass("gp2-st1").withDeleteClaim(true).withId(2).withSize("5000Gi").build()).build();
    // Volume added
    StorageDiff diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod, jbod2, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // Volume removed
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod2, jbod, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // Volume added with changes
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod, jbod3, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(false));
    assertThat(diff.shrinkSize(), is(true));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // No volume added, but with changes
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod2, jbod3, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(false));
    assertThat(diff.shrinkSize(), is(true));
    assertThat(diff.isVolumesAddedOrRemoved(), is(false));
    // Volume removed from the beginning
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod3, jbod5, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // Volume added to the beginning
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod5, jbod3, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // Volume replaced with another ID and another volume which is kept changed
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod3, jbod6, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(false));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // Volume replaced with another ID in single volume broker
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod, jbod4, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
    // Volume replaced with another ID without chenging the volumes which are kept
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod2, jbod6, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(true));
}

Example 29 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class StorageDiffTest method testJbodDiff.

@ParallelTest
public void testJbodDiff() {
    Storage jbod = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("100Gi").build(), new PersistentClaimStorageBuilder().withStorageClass("gp2-st1").withDeleteClaim(true).withId(1).withSize("1000Gi").build()).build();
    Storage jbod2 = new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(false).withId(0).withSize("1000Gi").build(), new PersistentClaimStorageBuilder().withStorageClass("gp2-ssd").withDeleteClaim(true).withId(1).withSize("1000Gi").build()).build();
    StorageDiff diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod, jbod, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(true));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(false));
    diff = new StorageDiff(Reconciliation.DUMMY_RECONCILIATION, jbod, jbod2, 3, 3);
    assertThat(diff.changesType(), is(false));
    assertThat(diff.isEmpty(), is(false));
    assertThat(diff.shrinkSize(), is(false));
    assertThat(diff.isVolumesAddedOrRemoved(), is(false));
}

Example 30 with JbodStorageBuilder

use of io.strimzi.api.kafka.model.storage.JbodStorageBuilder in project strimzi by strimzi.

the class KafkaST method testLabelsAndAnnotationForPVC.

@ParallelNamespaceTest
void testLabelsAndAnnotationForPVC(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String labelAnnotationKey = "testKey";
    final String firstValue = "testValue";
    final String changedValue = "editedTestValue";
    Map<String, String> pvcLabel = new HashMap<>();
    pvcLabel.put(labelAnnotationKey, firstValue);
    Map<String, String> pvcAnnotation = pvcLabel;
    Map<String, String> statefulSetLabels = new HashMap<>();
    statefulSetLabels.put("app.kubernetes.io/part-of", "some-app");
    statefulSetLabels.put("app.kubernetes.io/managed-by", "some-app");
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 1).editSpec().editKafka().withNewTemplate().withNewStatefulset().withNewMetadata().withLabels(statefulSetLabels).endMetadata().endStatefulset().withNewPodSet().withNewMetadata().withLabels(statefulSetLabels).endMetadata().endPodSet().withNewPersistentVolumeClaim().withNewMetadata().addToLabels(pvcLabel).addToAnnotations(pvcAnnotation).endMetadata().endPersistentVolumeClaim().endTemplate().withStorage(new JbodStorageBuilder().withVolumes(new PersistentClaimStorageBuilder().withDeleteClaim(false).withId(0).withSize("20Gi").build(), new PersistentClaimStorageBuilder().withDeleteClaim(true).withId(1).withSize("10Gi").build()).build()).endKafka().editZookeeper().withNewTemplate().withNewPersistentVolumeClaim().withNewMetadata().addToLabels(pvcLabel).addToAnnotations(pvcAnnotation).endMetadata().endPersistentVolumeClaim().endTemplate().withNewPersistentClaimStorage().withDeleteClaim(false).withSize("3Gi").endPersistentClaimStorage().endZookeeper().endSpec().build());
    LOGGER.info("Check if Kubernetes labels are applied");
    Map<String, String> actualStatefulSetLabels = StUtils.getLabelsOfStatefulSetOrStrimziPodSet(namespaceName, KafkaResources.kafkaStatefulSetName(clusterName));
    assertThat(actualStatefulSetLabels.get("app.kubernetes.io/part-of"), is("some-app"));
    assertThat(actualStatefulSetLabels.get("app.kubernetes.io/managed-by"), is("some-app"));
    LOGGER.info("Kubernetes labels are correctly set and present");
    List<PersistentVolumeClaim> pvcs = kubeClient(namespaceName).listPersistentVolumeClaims(namespaceName, clusterName).stream().filter(persistentVolumeClaim -> persistentVolumeClaim.getMetadata().getName().contains(clusterName)).collect(Collectors.toList());
    assertThat(pvcs.size(), is(7));
    for (PersistentVolumeClaim pvc : pvcs) {
        LOGGER.info("Verifying that PVC label {} - {} = {}", pvc.getMetadata().getName(), firstValue, pvc.getMetadata().getLabels().get(labelAnnotationKey));
        assertThat(firstValue, is(pvc.getMetadata().getLabels().get(labelAnnotationKey)));
        assertThat(firstValue, is(pvc.getMetadata().getAnnotations().get(labelAnnotationKey)));
    }
    pvcLabel.put(labelAnnotationKey, changedValue);
    pvcAnnotation.put(labelAnnotationKey, changedValue);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        LOGGER.info("Replacing kafka && zookeeper labels and annotations from {} to {}", labelAnnotationKey, changedValue);
        kafka.getSpec().getKafka().getTemplate().getPersistentVolumeClaim().getMetadata().setLabels(pvcLabel);
        kafka.getSpec().getKafka().getTemplate().getPersistentVolumeClaim().getMetadata().setAnnotations(pvcAnnotation);
        kafka.getSpec().getZookeeper().getTemplate().getPersistentVolumeClaim().getMetadata().setLabels(pvcLabel);
        kafka.getSpec().getZookeeper().getTemplate().getPersistentVolumeClaim().getMetadata().setAnnotations(pvcAnnotation);
    }, namespaceName);
    PersistentVolumeClaimUtils.waitUntilPVCLabelsChange(namespaceName, clusterName, pvcLabel, labelAnnotationKey);
    PersistentVolumeClaimUtils.waitUntilPVCAnnotationChange(namespaceName, clusterName, pvcAnnotation, labelAnnotationKey);
    KafkaUtils.waitForKafkaReady(namespaceName, clusterName);
    pvcs = kubeClient(namespaceName).listPersistentVolumeClaims(namespaceName, clusterName).stream().filter(persistentVolumeClaim -> persistentVolumeClaim.getMetadata().getName().contains(clusterName)).collect(Collectors.toList());
    LOGGER.info(pvcs.toString());
    assertThat(pvcs.size(), is(7));
    for (PersistentVolumeClaim pvc : pvcs) {
        LOGGER.info("Verifying replaced PVC label {} - {} = {}", pvc.getMetadata().getName(), firstValue, pvc.getMetadata().getLabels().get(labelAnnotationKey));
        assertThat(pvc.getMetadata().getLabels().get(labelAnnotationKey), is(changedValue));
        assertThat(pvc.getMetadata().getAnnotations().get(labelAnnotationKey), is(changedValue));
    }
}