Examples with KafkaClusterSpec io.strimzi.api.kafka.model.KafkaClusterSpec used on opensource projects

Example 6 with KafkaClusterSpec

use of io.strimzi.api.kafka.model.KafkaClusterSpec in project strimzi-kafka-operator by strimzi.

the class DynamicConfST method updateAndVerifyDynConf.

/**
 * UpdateAndVerifyDynConf, change the kafka configuration and verify that no rolling update were triggered
 * @param namespaceName name of the namespace
 * @param kafkaConfig specific kafka configuration, which will be changed
 */
private void updateAndVerifyDynConf(final String namespaceName, String clusterName, Map<String, Object> kafkaConfig) {
    LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    LOGGER.info("Updating configuration of Kafka cluster");
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        KafkaClusterSpec kafkaClusterSpec = k.getSpec().getKafka();
        kafkaClusterSpec.setConfig(kafkaConfig);
    }, namespaceName);
    PodUtils.verifyThatRunningPodsAreStable(namespaceName, KafkaResources.kafkaStatefulSetName(clusterName));
    assertThat(RollingUpdateUtils.componentHasRolled(namespaceName, kafkaSelector, kafkaPods), is(false));
}

Example 7 with KafkaClusterSpec

use of io.strimzi.api.kafka.model.KafkaClusterSpec in project strimzi-kafka-operator by strimzi.

the class KafkaCluster method fromCrd.

@SuppressWarnings({ "checkstyle:MethodLength", "checkstyle:JavaNCSS" })
public static KafkaCluster fromCrd(Reconciliation reconciliation, Kafka kafkaAssembly, KafkaVersion.Lookup versions, Storage oldStorage, int oldReplicas) {
    KafkaSpec kafkaSpec = kafkaAssembly.getSpec();
    KafkaClusterSpec kafkaClusterSpec = kafkaSpec.getKafka();
    KafkaCluster result = new KafkaCluster(reconciliation, kafkaAssembly);
    // This also validates that the Kafka version is supported
    result.kafkaVersion = versions.supportedVersion(kafkaClusterSpec.getVersion());
    result.setOwnerReference(kafkaAssembly);
    result.setReplicas(kafkaClusterSpec.getReplicas());
    validateIntConfigProperty("default.replication.factor", kafkaClusterSpec);
    validateIntConfigProperty("offsets.topic.replication.factor", kafkaClusterSpec);
    validateIntConfigProperty("transaction.state.log.replication.factor", kafkaClusterSpec);
    validateIntConfigProperty("transaction.state.log.min.isr", kafkaClusterSpec);
    result.setImage(versions.kafkaImage(kafkaClusterSpec.getImage(), kafkaClusterSpec.getVersion()));
    if (kafkaClusterSpec.getReadinessProbe() != null) {
        result.setReadinessProbe(kafkaClusterSpec.getReadinessProbe());
    }
    if (kafkaClusterSpec.getLivenessProbe() != null) {
        result.setLivenessProbe(kafkaClusterSpec.getLivenessProbe());
    }
    result.setRack(kafkaClusterSpec.getRack());
    String initImage = kafkaClusterSpec.getBrokerRackInitImage();
    if (initImage == null) {
        initImage = System.getenv().getOrDefault(ClusterOperatorConfig.STRIMZI_DEFAULT_KAFKA_INIT_IMAGE, "quay.io/strimzi/operator:latest");
    }
    result.setInitImage(initImage);
    Logging logging = kafkaClusterSpec.getLogging();
    result.setLogging(logging == null ? new InlineLogging() : logging);
    result.setGcLoggingEnabled(kafkaClusterSpec.getJvmOptions() == null ? DEFAULT_JVM_GC_LOGGING_ENABLED : kafkaClusterSpec.getJvmOptions().isGcLoggingEnabled());
    if (kafkaClusterSpec.getJvmOptions() != null) {
        result.setJavaSystemProperties(kafkaClusterSpec.getJvmOptions().getJavaSystemProperties());
    }
    result.setJvmOptions(kafkaClusterSpec.getJvmOptions());
    if (kafkaClusterSpec.getJmxOptions() != null) {
        result.setJmxEnabled(Boolean.TRUE);
        AuthenticationUtils.configureKafkaJmxOptions(kafkaClusterSpec.getJmxOptions().getAuthentication(), result);
    }
    // Handle Kafka broker configuration
    KafkaConfiguration configuration = new KafkaConfiguration(reconciliation, kafkaClusterSpec.getConfig().entrySet());
    configureCruiseControlMetrics(kafkaAssembly, result, configuration);
    validateConfiguration(reconciliation, kafkaAssembly, result.kafkaVersion, configuration);
    result.setConfiguration(configuration);
    // Parse different types of metrics configurations
    ModelUtils.parseMetrics(result, kafkaClusterSpec);
    if (oldStorage != null) {
        Storage newStorage = kafkaClusterSpec.getStorage();
        AbstractModel.validatePersistentStorage(newStorage);
        StorageDiff diff = new StorageDiff(reconciliation, oldStorage, newStorage, oldReplicas, kafkaClusterSpec.getReplicas());
        if (!diff.isEmpty()) {
            LOGGER.warnCr(reconciliation, "Only the following changes to Kafka storage are allowed: " + "changing the deleteClaim flag, " + "adding volumes to Jbod storage or removing volumes from Jbod storage, " + "changing overrides to nodes which do not exist yet" + "and increasing size of persistent claim volumes (depending on the volume type and used storage class).");
            LOGGER.warnCr(reconciliation, "The desired Kafka storage configuration in the custom resource {}/{} contains changes which are not allowed. As a " + "result, all storage changes will be ignored. Use DEBUG level logging for more information " + "about the detected changes.", kafkaAssembly.getMetadata().getNamespace(), kafkaAssembly.getMetadata().getName());
            Condition warning = StatusUtils.buildWarningCondition("KafkaStorage", "The desired Kafka storage configuration contains changes which are not allowed. As a " + "result, all storage changes will be ignored. Use DEBUG level logging for more information " + "about the detected changes.");
            result.addWarningCondition(warning);
            result.setStorage(oldStorage);
        } else {
            result.setStorage(newStorage);
        }
    } else {
        result.setStorage(kafkaClusterSpec.getStorage());
    }
    result.setResources(kafkaClusterSpec.getResources());
    // Configure listeners
    if (kafkaClusterSpec.getListeners() == null || kafkaClusterSpec.getListeners().isEmpty()) {
        LOGGER.errorCr(reconciliation, "The required field .spec.kafka.listeners is missing");
        throw new InvalidResourceException("The required field .spec.kafka.listeners is missing");
    }
    List<GenericKafkaListener> listeners = kafkaClusterSpec.getListeners();
    ListenersValidator.validate(reconciliation, kafkaClusterSpec.getReplicas(), listeners);
    result.setListeners(listeners);
    // Set authorization
    if (kafkaClusterSpec.getAuthorization() instanceof KafkaAuthorizationKeycloak) {
        if (!ListenersUtils.hasListenerWithOAuth(listeners)) {
            throw new InvalidResourceException("You cannot configure Keycloak Authorization without any listener with OAuth based authentication");
        } else {
            KafkaAuthorizationKeycloak authorizationKeycloak = (KafkaAuthorizationKeycloak) kafkaClusterSpec.getAuthorization();
            if (authorizationKeycloak.getClientId() == null || authorizationKeycloak.getTokenEndpointUri() == null) {
                LOGGER.errorCr(reconciliation, "Keycloak Authorization: Token Endpoint URI and clientId are both required");
                throw new InvalidResourceException("Keycloak Authorization: Token Endpoint URI and clientId are both required");
            }
        }
    }
    result.setAuthorization(kafkaClusterSpec.getAuthorization());
    if (kafkaClusterSpec.getTemplate() != null) {
        KafkaClusterTemplate template = kafkaClusterSpec.getTemplate();
        if (template.getStatefulset() != null) {
            if (template.getStatefulset().getPodManagementPolicy() != null) {
                result.templatePodManagementPolicy = template.getStatefulset().getPodManagementPolicy();
            }
            if (template.getStatefulset().getMetadata() != null) {
                result.templateStatefulSetLabels = template.getStatefulset().getMetadata().getLabels();
                result.templateStatefulSetAnnotations = template.getStatefulset().getMetadata().getAnnotations();
            }
        }
        if (template.getPodSet() != null && template.getPodSet().getMetadata() != null) {
            result.templatePodSetLabels = template.getPodSet().getMetadata().getLabels();
            result.templatePodSetAnnotations = template.getPodSet().getMetadata().getAnnotations();
        }
        ModelUtils.parsePodTemplate(result, template.getPod());
        ModelUtils.parseInternalServiceTemplate(result, template.getBootstrapService());
        ModelUtils.parseInternalHeadlessServiceTemplate(result, template.getBrokersService());
        if (template.getExternalBootstrapService() != null) {
            if (template.getExternalBootstrapService().getMetadata() != null) {
                result.templateExternalBootstrapServiceLabels = template.getExternalBootstrapService().getMetadata().getLabels();
                result.templateExternalBootstrapServiceAnnotations = template.getExternalBootstrapService().getMetadata().getAnnotations();
            }
        }
        if (template.getPerPodService() != null) {
            if (template.getPerPodService().getMetadata() != null) {
                result.templatePerPodServiceLabels = template.getPerPodService().getMetadata().getLabels();
                result.templatePerPodServiceAnnotations = template.getPerPodService().getMetadata().getAnnotations();
            }
        }
        if (template.getExternalBootstrapRoute() != null && template.getExternalBootstrapRoute().getMetadata() != null) {
            result.templateExternalBootstrapRouteLabels = template.getExternalBootstrapRoute().getMetadata().getLabels();
            result.templateExternalBootstrapRouteAnnotations = template.getExternalBootstrapRoute().getMetadata().getAnnotations();
        }
        if (template.getPerPodRoute() != null && template.getPerPodRoute().getMetadata() != null) {
            result.templatePerPodRouteLabels = template.getPerPodRoute().getMetadata().getLabels();
            result.templatePerPodRouteAnnotations = template.getPerPodRoute().getMetadata().getAnnotations();
        }
        if (template.getExternalBootstrapIngress() != null && template.getExternalBootstrapIngress().getMetadata() != null) {
            result.templateExternalBootstrapIngressLabels = template.getExternalBootstrapIngress().getMetadata().getLabels();
            result.templateExternalBootstrapIngressAnnotations = template.getExternalBootstrapIngress().getMetadata().getAnnotations();
        }
        if (template.getPerPodIngress() != null && template.getPerPodIngress().getMetadata() != null) {
            result.templatePerPodIngressLabels = template.getPerPodIngress().getMetadata().getLabels();
            result.templatePerPodIngressAnnotations = template.getPerPodIngress().getMetadata().getAnnotations();
        }
        if (template.getClusterRoleBinding() != null && template.getClusterRoleBinding().getMetadata() != null) {
            result.templateClusterRoleBindingLabels = template.getClusterRoleBinding().getMetadata().getLabels();
            result.templateClusterRoleBindingAnnotations = template.getClusterRoleBinding().getMetadata().getAnnotations();
        }
        if (template.getPersistentVolumeClaim() != null && template.getPersistentVolumeClaim().getMetadata() != null) {
            result.templatePersistentVolumeClaimLabels = Util.mergeLabelsOrAnnotations(template.getPersistentVolumeClaim().getMetadata().getLabels(), result.templateStatefulSetLabels);
            result.templatePersistentVolumeClaimAnnotations = template.getPersistentVolumeClaim().getMetadata().getAnnotations();
        }
        if (template.getKafkaContainer() != null && template.getKafkaContainer().getEnv() != null) {
            result.templateKafkaContainerEnvVars = template.getKafkaContainer().getEnv();
        }
        if (template.getInitContainer() != null && template.getInitContainer().getEnv() != null) {
            result.templateInitContainerEnvVars = template.getInitContainer().getEnv();
        }
        if (template.getKafkaContainer() != null && template.getKafkaContainer().getSecurityContext() != null) {
            result.templateKafkaContainerSecurityContext = template.getKafkaContainer().getSecurityContext();
        }
        if (template.getInitContainer() != null && template.getInitContainer().getSecurityContext() != null) {
            result.templateInitContainerSecurityContext = template.getInitContainer().getSecurityContext();
        }
        if (template.getServiceAccount() != null && template.getServiceAccount().getMetadata() != null) {
            result.templateServiceAccountLabels = template.getServiceAccount().getMetadata().getLabels();
            result.templateServiceAccountAnnotations = template.getServiceAccount().getMetadata().getAnnotations();
        }
        if (template.getJmxSecret() != null && template.getJmxSecret().getMetadata() != null) {
            result.templateJmxSecretLabels = template.getJmxSecret().getMetadata().getLabels();
            result.templateJmxSecretAnnotations = template.getJmxSecret().getMetadata().getAnnotations();
        }
        ModelUtils.parsePodDisruptionBudgetTemplate(result, template.getPodDisruptionBudget());
    }
    result.templatePodLabels = Util.mergeLabelsOrAnnotations(result.templatePodLabels, DEFAULT_POD_LABELS);
    return result;
}

Example 8 with KafkaClusterSpec

use of io.strimzi.api.kafka.model.KafkaClusterSpec in project strimzi-kafka-operator by strimzi.

the class KafkaExporter method fromCrd.

public static KafkaExporter fromCrd(Reconciliation reconciliation, Kafka kafkaAssembly, KafkaVersion.Lookup versions) {
    KafkaExporter kafkaExporter = new KafkaExporter(reconciliation, kafkaAssembly);
    KafkaExporterSpec spec = kafkaAssembly.getSpec().getKafkaExporter();
    if (spec != null) {
        kafkaExporter.isDeployed = true;
        kafkaExporter.setResources(spec.getResources());
        if (spec.getReadinessProbe() != null) {
            kafkaExporter.setReadinessProbe(spec.getReadinessProbe());
        }
        if (spec.getLivenessProbe() != null) {
            kafkaExporter.setLivenessProbe(spec.getLivenessProbe());
        }
        kafkaExporter.setGroupRegex(spec.getGroupRegex());
        kafkaExporter.setTopicRegex(spec.getTopicRegex());
        String image = spec.getImage();
        if (image == null) {
            KafkaClusterSpec kafkaClusterSpec = kafkaAssembly.getSpec().getKafka();
            image = System.getenv().getOrDefault(ClusterOperatorConfig.STRIMZI_DEFAULT_KAFKA_EXPORTER_IMAGE, versions.kafkaImage(kafkaClusterSpec.getImage(), versions.defaultVersion().version()));
        }
        kafkaExporter.setImage(image);
        kafkaExporter.setLogging(spec.getLogging());
        kafkaExporter.setSaramaLoggingEnabled(spec.getEnableSaramaLogging());
        if (spec.getTemplate() != null) {
            KafkaExporterTemplate template = spec.getTemplate();
            if (template.getDeployment() != null && template.getDeployment().getMetadata() != null) {
                kafkaExporter.templateDeploymentLabels = template.getDeployment().getMetadata().getLabels();
                kafkaExporter.templateDeploymentAnnotations = template.getDeployment().getMetadata().getAnnotations();
            }
            if (template.getContainer() != null && template.getContainer().getEnv() != null) {
                kafkaExporter.templateContainerEnvVars = template.getContainer().getEnv();
            }
            if (template.getContainer() != null && template.getContainer().getSecurityContext() != null) {
                kafkaExporter.templateContainerSecurityContext = template.getContainer().getSecurityContext();
            }
            if (template.getServiceAccount() != null && template.getServiceAccount().getMetadata() != null) {
                kafkaExporter.templateServiceAccountLabels = template.getServiceAccount().getMetadata().getLabels();
                kafkaExporter.templateServiceAccountAnnotations = template.getServiceAccount().getMetadata().getAnnotations();
            }
            ModelUtils.parsePodTemplate(kafkaExporter, template.getPod());
        }
        kafkaExporter.setVersion(versions.supportedVersion(kafkaAssembly.getSpec().getKafka().getVersion()).version());
        kafkaExporter.setOwnerReference(kafkaAssembly);
    } else {
        kafkaExporter.isDeployed = false;
    }
    kafkaExporter.templatePodLabels = Util.mergeLabelsOrAnnotations(kafkaExporter.templatePodLabels, DEFAULT_POD_LABELS);
    return kafkaExporter;
}

Example 9 with KafkaClusterSpec

use of io.strimzi.api.kafka.model.KafkaClusterSpec in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class IngressControllerManager method summarize.

static LongSummaryStatistics summarize(List<Kafka> managedKafkas, Function<Kafka, String> quantity, Supplier<String> defaultValue) {
    return managedKafkas.stream().map(m -> {
        KafkaClusterSpec s = m.getSpec().getKafka();
        String value = Optional.of(quantity.apply(m)).orElseGet(defaultValue);
        return Quantity.getAmountInBytes(Quantity.parse(value)).multiply(BigDecimal.valueOf(s.getReplicas()));
    }).mapToLong(BigDecimal::longValue).summaryStatistics();
}

Example 10 with KafkaClusterSpec

use of io.strimzi.api.kafka.model.KafkaClusterSpec in project strimzi by strimzi.

the class ResourceUtils method createKafka.

public static Kafka createKafka(String namespace, String name, int replicas, String image, int healthDelay, int healthTimeout, MetricsConfig metricsConfig, Map<String, Object> kafkaConfiguration, Map<String, Object> zooConfiguration, Storage kafkaStorage, SingleVolumeStorage zkStorage, Logging kafkaLogging, Logging zkLogging, KafkaExporterSpec keSpec, CruiseControlSpec ccSpec) {
    Kafka result = new Kafka();
    ObjectMeta meta = new ObjectMetaBuilder().withNamespace(namespace).withName(name).withLabels(Labels.fromMap(TestUtils.map(Labels.KUBERNETES_DOMAIN + "part-of", "tests", "my-user-label", "cromulent")).toMap()).build();
    result.setMetadata(meta);
    KafkaSpec spec = new KafkaSpec();
    KafkaClusterSpec kafkaClusterSpec = new KafkaClusterSpec();
    kafkaClusterSpec.setReplicas(replicas);
    kafkaClusterSpec.setListeners(singletonList(new GenericKafkaListenerBuilder().withName("plain").withPort(9092).withTls(false).withType(KafkaListenerType.INTERNAL).build()));
    kafkaClusterSpec.setImage(image);
    if (kafkaLogging != null) {
        kafkaClusterSpec.setLogging(kafkaLogging);
    }
    Probe livenessProbe = new Probe();
    livenessProbe.setInitialDelaySeconds(healthDelay);
    livenessProbe.setTimeoutSeconds(healthTimeout);
    livenessProbe.setSuccessThreshold(4);
    livenessProbe.setFailureThreshold(10);
    livenessProbe.setPeriodSeconds(33);
    kafkaClusterSpec.setLivenessProbe(livenessProbe);
    kafkaClusterSpec.setReadinessProbe(livenessProbe);
    kafkaClusterSpec.setMetricsConfig(metricsConfig);
    if (kafkaConfiguration != null) {
        kafkaClusterSpec.setConfig(kafkaConfiguration);
    }
    kafkaClusterSpec.setStorage(kafkaStorage);
    spec.setKafka(kafkaClusterSpec);
    ZookeeperClusterSpec zk = new ZookeeperClusterSpec();
    zk.setReplicas(replicas);
    zk.setImage(image + "-zk");
    if (zkLogging != null) {
        zk.setLogging(zkLogging);
    }
    zk.setLivenessProbe(livenessProbe);
    zk.setReadinessProbe(livenessProbe);
    if (zooConfiguration != null) {
        zk.setConfig(zooConfiguration);
    }
    zk.setStorage(zkStorage);
    zk.setMetricsConfig(metricsConfig);
    spec.setKafkaExporter(keSpec);
    spec.setCruiseControl(ccSpec);
    spec.setZookeeper(zk);
    result.setSpec(spec);
    return result;
}