Search in sources :

Example 6 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class KafkaAssemblyOperatorMockTest method testReconcileUpdatesKafkaPersistentVolumes.

@ParameterizedTest
@MethodSource("data")
public void testReconcileUpdatesKafkaPersistentVolumes(Params params, VertxTestContext context) {
    init(params);
    assumeTrue(kafkaStorage instanceof PersistentClaimStorage, "Parameterized Test only runs for Params with Kafka Persistent storage");
    String originalStorageClass = Storage.storageClass(kafkaStorage);
    Checkpoint async = context.checkpoint();
    initialReconcile(context).onComplete(context.succeeding(v -> context.verify(() -> {
        assertStorageClass(context, KafkaCluster.kafkaClusterName(CLUSTER_NAME), originalStorageClass);
        // Try to update the storage class
        String changedClass = originalStorageClass + "2";
        Kafka patchedPersistenceKafka = new KafkaBuilder(cluster).editSpec().editKafka().withNewPersistentClaimStorage().withStorageClass(changedClass).withSize("123").endPersistentClaimStorage().endKafka().endSpec().build();
        kafkaAssembly(NAMESPACE, CLUSTER_NAME).patch(patchedPersistenceKafka);
        LOGGER.info("Updating with changed storage class");
    }))).compose(v -> operator.reconcile(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME))).onComplete(context.succeeding(v -> context.verify(() -> {
        // Check the storage class was not changed
        assertStorageClass(context, KafkaCluster.kafkaClusterName(CLUSTER_NAME), originalStorageClass);
        async.flag();
    })));
}
Also used : Quantity(io.fabric8.kubernetes.api.model.Quantity) CoreMatchers.is(org.hamcrest.CoreMatchers.is) DeletionPropagation(io.fabric8.kubernetes.api.model.DeletionPropagation) Storage(io.strimzi.api.kafka.model.storage.Storage) StatefulSetStatus(io.fabric8.kubernetes.api.model.apps.StatefulSetStatus) PodTemplateSpec(io.fabric8.kubernetes.api.model.PodTemplateSpec) MockKube(io.strimzi.test.mockkube.MockKube) CoreMatchers.notNullValue(org.hamcrest.CoreMatchers.notNullValue) AfterAll(org.junit.jupiter.api.AfterAll) Resource(io.fabric8.kubernetes.client.dsl.Resource) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) ResourceRequirements(io.fabric8.kubernetes.api.model.ResourceRequirements) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Ca(io.strimzi.operator.cluster.model.Ca) BeforeAll(org.junit.jupiter.api.BeforeAll) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) ResourceOperatorSupplier(io.strimzi.operator.cluster.operator.resource.ResourceOperatorSupplier) ResourceUtils(io.strimzi.operator.cluster.ResourceUtils) AbstractModel(io.strimzi.operator.cluster.model.AbstractModel) StatefulSetOperator(io.strimzi.operator.cluster.operator.resource.StatefulSetOperator) MethodSource(org.junit.jupiter.params.provider.MethodSource) KafkaVersion(io.strimzi.operator.cluster.model.KafkaVersion) Set(java.util.Set) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) VertxExtension(io.vertx.junit5.VertxExtension) Future(io.vertx.core.Future) Collectors(java.util.stream.Collectors) StatefulSetSpec(io.fabric8.kubernetes.api.model.apps.StatefulSetSpec) List(java.util.List) EphemeralStorage(io.strimzi.api.kafka.model.storage.EphemeralStorage) KafkaConfiguration(io.strimzi.operator.cluster.model.KafkaConfiguration) Labels(io.strimzi.operator.common.model.Labels) Logger(org.apache.logging.log4j.Logger) StrimziPodSet(io.strimzi.api.kafka.model.StrimziPodSet) PasswordGenerator(io.strimzi.operator.common.PasswordGenerator) PersistentVolumeClaim(io.fabric8.kubernetes.api.model.PersistentVolumeClaim) Optional(java.util.Optional) Checkpoint(io.vertx.junit5.Checkpoint) PlatformFeaturesAvailability(io.strimzi.operator.PlatformFeaturesAvailability) ClusterOperatorConfig(io.strimzi.operator.cluster.ClusterOperatorConfig) MockCertManager(io.strimzi.operator.common.operator.MockCertManager) VertxTestContext(io.vertx.junit5.VertxTestContext) KafkaList(io.strimzi.api.kafka.KafkaList) Matchers.aMapWithSize(org.hamcrest.Matchers.aMapWithSize) Container(io.fabric8.kubernetes.api.model.Container) ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SingleVolumeStorage(io.strimzi.api.kafka.model.storage.SingleVolumeStorage) HashMap(java.util.HashMap) Storage.deleteClaim(io.strimzi.api.kafka.model.storage.Storage.deleteClaim) Crds(io.strimzi.api.kafka.Crds) ZookeeperCluster(io.strimzi.operator.cluster.model.ZookeeperCluster) AtomicReference(java.util.concurrent.atomic.AtomicReference) KafkaBuilder(io.strimzi.api.kafka.model.KafkaBuilder) FeatureGates(io.strimzi.operator.cluster.FeatureGates) ArrayList(java.util.ArrayList) PersistentClaimStorageBuilder(io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder) KafkaCluster(io.strimzi.operator.cluster.model.KafkaCluster) ClusterOperator(io.strimzi.operator.cluster.ClusterOperator) KafkaVersionTestUtils(io.strimzi.operator.cluster.KafkaVersionTestUtils) PodSpec(io.fabric8.kubernetes.api.model.PodSpec) ZookeeperLeaderFinder(io.strimzi.operator.cluster.operator.resource.ZookeeperLeaderFinder) Assumptions.assumeTrue(org.junit.jupiter.api.Assumptions.assumeTrue) Matchers.hasSize(org.hamcrest.Matchers.hasSize) CustomResourceDefinition(io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition) Collections.singletonMap(java.util.Collections.singletonMap) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Volume(io.fabric8.kubernetes.api.model.Volume) Matchers.hasEntry(org.hamcrest.Matchers.hasEntry) StrimziPodSetList(io.strimzi.api.kafka.StrimziPodSetList) KubernetesVersion(io.strimzi.operator.KubernetesVersion) Vertx(io.vertx.core.Vertx) StatefulSet(io.fabric8.kubernetes.api.model.apps.StatefulSet) Reconciliation(io.strimzi.operator.common.Reconciliation) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) KafkaListenerType(io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType) KubernetesClient(io.fabric8.kubernetes.client.KubernetesClient) Kafka(io.strimzi.api.kafka.model.Kafka) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) Checkpoint(io.vertx.junit5.Checkpoint) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) Reconciliation(io.strimzi.operator.common.Reconciliation) Kafka(io.strimzi.api.kafka.model.Kafka) KafkaBuilder(io.strimzi.api.kafka.model.KafkaBuilder) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 7 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class CruiseControlTest method testBrokerCapacities.

@ParallelTest
public void testBrokerCapacities() {
    // Test user defined capacities
    BrokerCapacity userDefinedBrokerCapacity = new BrokerCapacity();
    userDefinedBrokerCapacity.setInboundNetwork("50000KB/s");
    userDefinedBrokerCapacity.setOutboundNetwork("50000KB/s");
    CruiseControlSpec cruiseControlSpec = new CruiseControlSpecBuilder().withImage(ccImage).withBrokerCapacity(userDefinedBrokerCapacity).build();
    Kafka resource = createKafka(cruiseControlSpec);
    Capacity capacity = new Capacity(resource.getSpec(), kafkaStorage);
    assertThat(getCapacityConfigurationFromEnvVar(resource, ENV_VAR_CRUISE_CONTROL_CAPACITY_CONFIGURATION), is(capacity.generateCapacityConfig()));
    // Test generated disk capacity
    JbodStorage jbodStorage = new JbodStorage();
    List<SingleVolumeStorage> volumes = new ArrayList<>();
    PersistentClaimStorage p1 = new PersistentClaimStorage();
    p1.setId(0);
    p1.setSize("50Gi");
    volumes.add(p1);
    PersistentClaimStorage p2 = new PersistentClaimStorage();
    p2.setId(1);
    volumes.add(p2);
    jbodStorage.setVolumes(volumes);
    resource = new KafkaBuilder(ResourceUtils.createKafka(namespace, cluster, replicas, image, healthDelay, healthTimeout)).editSpec().editKafka().withVersion(version).withStorage(jbodStorage).endKafka().withCruiseControl(cruiseControlSpec).endSpec().build();
    capacity = new Capacity(resource.getSpec(), jbodStorage);
    assertThat(getCapacityConfigurationFromEnvVar(resource, ENV_VAR_CRUISE_CONTROL_CAPACITY_CONFIGURATION), is(capacity.generateCapacityConfig()));
}
Also used : CruiseControlSpec(io.strimzi.api.kafka.model.CruiseControlSpec) BrokerCapacity(io.strimzi.api.kafka.model.balancing.BrokerCapacity) Capacity(io.strimzi.operator.cluster.model.cruisecontrol.Capacity) BrokerCapacity(io.strimzi.api.kafka.model.balancing.BrokerCapacity) SingleVolumeStorage(io.strimzi.api.kafka.model.storage.SingleVolumeStorage) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) Kafka(io.strimzi.api.kafka.model.Kafka) ArrayList(java.util.ArrayList) CruiseControlSpecBuilder(io.strimzi.api.kafka.model.CruiseControlSpecBuilder) KafkaBuilder(io.strimzi.api.kafka.model.KafkaBuilder) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage) ParallelTest(io.strimzi.test.annotations.ParallelTest)

Example 8 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class Capacity method generateJbodDiskCapacity.

/**
 * Generate JBOD disk capacity configuration for a broker using the supplied storage configuration
 *
 * @param storage Storage configuration for Kafka cluster
 * @param idx Index of the broker
 * @return Disk capacity configuration value as a JsonObject for broker idx
 */
private JsonObject generateJbodDiskCapacity(Storage storage, int idx) {
    JsonObject json = new JsonObject();
    String size = "";
    for (SingleVolumeStorage volume : ((JbodStorage) storage).getVolumes()) {
        String name = VolumeUtils.createVolumePrefix(volume.getId(), true);
        String path = AbstractModel.KAFKA_MOUNT_PATH + "/" + name + "/" + AbstractModel.KAFKA_LOG_DIR + idx;
        if (volume instanceof PersistentClaimStorage) {
            size = ((PersistentClaimStorage) volume).getSize();
        } else if (volume instanceof EphemeralStorage) {
            size = ((EphemeralStorage) volume).getSizeLimit();
        }
        json.put(path, String.valueOf(Capacity.getSizeInMiB(size)));
    }
    return json;
}
Also used : SingleVolumeStorage(io.strimzi.api.kafka.model.storage.SingleVolumeStorage) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) JsonObject(io.vertx.core.json.JsonObject) EphemeralStorage(io.strimzi.api.kafka.model.storage.EphemeralStorage) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage)

Example 9 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project strimzi-kafka-operator by strimzi.

the class StorageDiff method isOverrideChangeAllowed.

/**
 * Validates the changes to the storage overrides and decides whether they are allowed or not. Allowed changes are
 * those to nodes which will be added, removed or which do nto exist yet.
 *
 * @param current           Current Storage configuration
 * @param desired           New storage configuration
 * @param currentReplicas   Current number of replicas
 * @param desiredReplicas   Desired number of replicas
 * @return                  True if only allowed override changes were done, false othewise
 */
private boolean isOverrideChangeAllowed(Storage current, Storage desired, int currentReplicas, int desiredReplicas) {
    List<PersistentClaimStorageOverride> currentOverrides = ((PersistentClaimStorage) current).getOverrides();
    if (currentOverrides == null) {
        currentOverrides = Collections.emptyList();
    }
    List<PersistentClaimStorageOverride> desiredOverrides = ((PersistentClaimStorage) desired).getOverrides();
    if (desiredOverrides == null) {
        desiredOverrides = Collections.emptyList();
    }
    // We care only about the nodes which existed before this reconciliation and will still exist after it
    int existedAndWillExist = Math.min(currentReplicas, desiredReplicas);
    for (int i = 0; i < existedAndWillExist; i++) {
        int nodeId = i;
        PersistentClaimStorageOverride currentOverride = currentOverrides.stream().filter(override -> override.getBroker() == nodeId).findFirst().orElse(null);
        PersistentClaimStorageOverride desiredOverride = desiredOverrides.stream().filter(override -> override.getBroker() == nodeId).findFirst().orElse(null);
        if (currentOverride != null && desiredOverride != null) {
            // Both overrides exist but are not equal
            if (!currentOverride.equals(desiredOverride)) {
                return false;
            }
        } else if (currentOverride != null || desiredOverride != null) {
            // One of them is null while the other is not null => they differ
            return false;
        }
    }
    return true;
}
Also used : PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) PersistentClaimStorageOverride(io.strimzi.api.kafka.model.storage.PersistentClaimStorageOverride)

Example 10 with PersistentClaimStorage

use of io.strimzi.api.kafka.model.storage.PersistentClaimStorage in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaCluster method getAdjustedMaxDataRetentionSize.

/**
 * Get the effective volume size considering extra padding and the existing size
 */
private Quantity getAdjustedMaxDataRetentionSize(ManagedKafka managedKafka, Kafka current) {
    long bytes = getPerBrokerBytes(managedKafka, current, managedKafka.getSpec().getCapacity().getMaxDataRetentionSize(), () -> this.config.getKafka().getVolumeSize());
    // pad to give a margin before soft/hard limits kick in
    bytes += getStoragePadding(managedKafka, current);
    // strimzi won't allow the size to be reduced so scrape the size if possible
    if (current != null) {
        Storage storage = current.getSpec().getKafka().getStorage();
        if (storage instanceof JbodStorage) {
            JbodStorage jbodStorage = (JbodStorage) storage;
            for (SingleVolumeStorage singleVolumeStorage : jbodStorage.getVolumes()) {
                if (singleVolumeStorage instanceof PersistentClaimStorage && Integer.valueOf(JBOD_VOLUME_ID).equals(singleVolumeStorage.getId())) {
                    String existingSize = ((PersistentClaimStorage) singleVolumeStorage).getSize();
                    long existingBytes = Quantity.getAmountInBytes(Quantity.parse(existingSize)).longValue();
                    // TODO: if not changed a warning may be appropriate, but it would be best as a status condition
                    bytes = Math.max(existingBytes, bytes);
                    break;
                }
            }
        }
    }
    return new Quantity(String.valueOf(bytes));
}
Also used : Storage(io.strimzi.api.kafka.model.storage.Storage) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) SingleVolumeStorage(io.strimzi.api.kafka.model.storage.SingleVolumeStorage) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage) SingleVolumeStorage(io.strimzi.api.kafka.model.storage.SingleVolumeStorage) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) Quantity(io.fabric8.kubernetes.api.model.Quantity) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage)

Aggregations

PersistentClaimStorage (io.strimzi.api.kafka.model.storage.PersistentClaimStorage)30 SingleVolumeStorage (io.strimzi.api.kafka.model.storage.SingleVolumeStorage)21 JbodStorage (io.strimzi.api.kafka.model.storage.JbodStorage)19 ArrayList (java.util.ArrayList)15 PersistentVolumeClaim (io.fabric8.kubernetes.api.model.PersistentVolumeClaim)13 EphemeralStorage (io.strimzi.api.kafka.model.storage.EphemeralStorage)12 Kafka (io.strimzi.api.kafka.model.Kafka)11 KafkaBuilder (io.strimzi.api.kafka.model.KafkaBuilder)11 PersistentClaimStorageBuilder (io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder)11 GenericKafkaListenerBuilder (io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder)10 Checkpoint (io.vertx.junit5.Checkpoint)10 KubernetesClient (io.fabric8.kubernetes.client.KubernetesClient)9 Storage (io.strimzi.api.kafka.model.storage.Storage)9 Volume (io.fabric8.kubernetes.api.model.Volume)8 CustomResourceDefinition (io.fabric8.kubernetes.api.model.apiextensions.v1.CustomResourceDefinition)8 Crds (io.strimzi.api.kafka.Crds)8 KafkaList (io.strimzi.api.kafka.KafkaList)8 StrimziPodSetList (io.strimzi.api.kafka.StrimziPodSetList)8 StrimziPodSet (io.strimzi.api.kafka.model.StrimziPodSet)8 KafkaListenerType (io.strimzi.api.kafka.model.listener.arraylistener.KafkaListenerType)8