Examples with PersistentClaimStorage io.strimzi.api.kafka.model.storage.PersistentClaimStorage used on opensource projects

Example 11 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaClusterTest method getBrokerStorageBytes.

private long getBrokerStorageBytes(Kafka kafka) {
    JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
    PersistentClaimStorage pcs = (PersistentClaimStorage) storage.getVolumes().get(0);
    Quantity quantity = Quantity.parse(pcs.getSize());
    return Quantity.getAmountInBytes(quantity).longValue();
}

Example 12 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi by strimzi.

the class CruiseControlTest method testBrokerCapacities.

@ParallelTest
public void testBrokerCapacities() {
    // Test user defined capacities
    BrokerCapacity userDefinedBrokerCapacity = new BrokerCapacity();
    userDefinedBrokerCapacity.setInboundNetwork("50000KB/s");
    userDefinedBrokerCapacity.setOutboundNetwork("50000KB/s");
    CruiseControlSpec cruiseControlSpec = new CruiseControlSpecBuilder().withImage(ccImage).withBrokerCapacity(userDefinedBrokerCapacity).build();
    Kafka resource = createKafka(cruiseControlSpec);
    Capacity capacity = new Capacity(resource.getSpec(), kafkaStorage);
    assertThat(getCapacityConfigurationFromEnvVar(resource, ENV_VAR_CRUISE_CONTROL_CAPACITY_CONFIGURATION), is(capacity.generateCapacityConfig()));
    // Test generated disk capacity
    JbodStorage jbodStorage = new JbodStorage();
    List<SingleVolumeStorage> volumes = new ArrayList<>();
    PersistentClaimStorage p1 = new PersistentClaimStorage();
    p1.setId(0);
    p1.setSize("50Gi");
    volumes.add(p1);
    PersistentClaimStorage p2 = new PersistentClaimStorage();
    p2.setId(1);
    volumes.add(p2);
    jbodStorage.setVolumes(volumes);
    resource = new KafkaBuilder(ResourceUtils.createKafka(namespace, cluster, replicas, image, healthDelay, healthTimeout)).editSpec().editKafka().withVersion(version).withStorage(jbodStorage).endKafka().withCruiseControl(cruiseControlSpec).endSpec().build();
    capacity = new Capacity(resource.getSpec(), jbodStorage);
    assertThat(getCapacityConfigurationFromEnvVar(resource, ENV_VAR_CRUISE_CONTROL_CAPACITY_CONFIGURATION), is(capacity.generateCapacityConfig()));
}

Example 13 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi by strimzi.

the class KafkaAssemblyOperatorMockTest method testReconcileUpdatesKafkaPersistentVolumes.

@ParameterizedTest
@MethodSource("data")
public void testReconcileUpdatesKafkaPersistentVolumes(Params params, VertxTestContext context) {
    init(params);
    assumeTrue(kafkaStorage instanceof PersistentClaimStorage, "Parameterized Test only runs for Params with Kafka Persistent storage");
    String originalStorageClass = Storage.storageClass(kafkaStorage);
    Checkpoint async = context.checkpoint();
    initialReconcile(context).onComplete(context.succeeding(v -> context.verify(() -> {
        assertStorageClass(context, KafkaCluster.kafkaClusterName(CLUSTER_NAME), originalStorageClass);
        // Try to update the storage class
        String changedClass = originalStorageClass + "2";
        Kafka patchedPersistenceKafka = new KafkaBuilder(cluster).editSpec().editKafka().withNewPersistentClaimStorage().withStorageClass(changedClass).withSize("123").endPersistentClaimStorage().endKafka().endSpec().build();
        kafkaAssembly(NAMESPACE, CLUSTER_NAME).patch(patchedPersistenceKafka);
        LOGGER.info("Updating with changed storage class");
    }))).compose(v -> operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME))).onComplete(context.succeeding(v -> context.verify(() -> {
        // Check the storage class was not changed
        assertStorageClass(context, KafkaCluster.kafkaClusterName(CLUSTER_NAME), originalStorageClass);
        async.flag();
    })));
}

Example 14 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi by strimzi.

the class JbodStorageTest method testJbodStorageCreatesPersistentVolumeClaimsMatchingKafkaVolumes.

@Test
public void testJbodStorageCreatesPersistentVolumeClaimsMatchingKafkaVolumes(VertxTestContext context) {
    Checkpoint async = context.checkpoint();
    operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, NAME)).onComplete(context.succeeding(v -> context.verify(() -> {
        List<PersistentVolumeClaim> pvcs = getPvcs(NAMESPACE, NAME);
        for (int i = 0; i < this.kafka.getSpec().getKafka().getReplicas(); i++) {
            int podId = i;
            for (SingleVolumeStorage volume : this.volumes) {
                if (volume instanceof PersistentClaimStorage) {
                    String expectedPvcName = VolumeUtils.createVolumePrefix(volume.getId(), true) + "-" + KafkaCluster.kafkaPodName(NAME, podId);
                    List<PersistentVolumeClaim> matchingPvcs = pvcs.stream().filter(pvc -> pvc.getMetadata().getName().equals(expectedPvcName)).collect(Collectors.toList());
                    assertThat("Exactly one pvc should have the name " + expectedPvcName + " in :\n" + pvcs.toString(), matchingPvcs, Matchers.hasSize(1));
                    PersistentVolumeClaim pvc = matchingPvcs.get(0);
                    boolean isDeleteClaim = ((PersistentClaimStorage) volume).isDeleteClaim();
                    assertThat("deleteClaim value did not match for volume : " + volume.toString(), Annotations.booleanAnnotation(pvc, AbstractModel.ANNO_STRIMZI_IO_DELETE_CLAIM, false), is(isDeleteClaim));
                }
            }
        }
        async.flag();
    })));
}

Example 15 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi by strimzi.

the class AlternativeReconcileTriggersST method testAddingAndRemovingJbodVolumes.

/**
 * Adding and removing JBOD volumes requires rolling updates in the sequential order. Otherwise the StatefulSet does
 * not like it. This tests tries to add and remove volume from JBOD to test both of these situations.
 */
@ParallelNamespaceTest
void testAddingAndRemovingJbodVolumes(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    final String continuousTopicName = "continuous-topic";
    // 500 messages will take 500 seconds in that case
    final int continuousClientsMessageCount = 500;
    final String producerName = "hello-world-producer";
    final String consumerName = "hello-world-consumer";
    PersistentClaimStorage vol0 = new PersistentClaimStorageBuilder().withId(0).withSize("1Gi").withDeleteClaim(true).build();
    PersistentClaimStorage vol1 = new PersistentClaimStorageBuilder().withId(1).withSize("1Gi").withDeleteClaim(true).build();
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaJBOD(clusterName, 3, 3, new JbodStorageBuilder().addToVolumes(vol0).build()).build());
    final String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, topicName).build());
    // ##############################
    // Attach clients which will continuously produce/consume messages to/from Kafka brokers during rolling update
    // ##############################
    // Setup topic, which has 3 replicas and 2 min.isr to see if producer will be able to work during rolling update
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(clusterName, continuousTopicName, 3, 3, 2).build());
    String producerAdditionConfiguration = "delivery.timeout.ms=20000\nrequest.timeout.ms=20000";
    // Add transactional id to make producer transactional
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\ntransactional.id=" + continuousTopicName + ".1");
    producerAdditionConfiguration = producerAdditionConfiguration.concat("\nenable.idempotence=true");
    KafkaClients kafkaBasicClientJob = new KafkaClientsBuilder().withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(clusterName)).withTopicName(continuousTopicName).withMessageCount(continuousClientsMessageCount).withAdditionalConfig(producerAdditionConfiguration).withDelayMs(1000).withNamespaceName(namespaceName).build();
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.producerStrimzi());
    resourceManager.createResource(extensionContext, kafkaBasicClientJob.consumerStrimzi());
    // ##############################
    String userName = KafkaUserUtils.generateRandomNameOfKafkaUser();
    KafkaUser user = KafkaUserTemplates.tlsUser(clusterName, userName).build();
    resourceManager.createResource(extensionContext, user);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(true, kafkaClientsName, user).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicName).withNamespaceName(namespaceName).withClusterName(clusterName).withMessageCount(MESSAGE_COUNT).withKafkaUsername(userName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient.produceTlsMessagesUntilOperationIsSuccessful(MESSAGE_COUNT);
    // Add Jbod volume to Kafka => triggers RU
    LOGGER.info("Add JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().add(vol1);
    }, namespaceName);
    // Wait util it rolls
    kafkaPods = RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // Remove Jbod volume to Kafka => triggers RU
    LOGGER.info("Remove JBOD volume to the Kafka cluster {}", kafkaName);
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, kafka -> {
        JbodStorage storage = (JbodStorage) kafka.getSpec().getKafka().getStorage();
        storage.getVolumes().remove(vol1);
    }, namespaceName);
    // Wait util it rolls
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, 3, kafkaPods);
    // ##############################
    // Validate that continuous clients finished successfully
    // ##############################
    ClientUtils.waitTillContinuousClientsFinish(producerName, consumerName, namespaceName, continuousClientsMessageCount);
// ##############################
}