Search in sources :

Example 1 with KafkaSpec

use of org.bf2.cos.fleetshard.api.KafkaSpec in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorDeploymentProvisioner method createManagedConnector.

private ManagedConnector createManagedConnector(String uow, ConnectorDeployment deployment, HasMetadata owner) {
    ManagedConnector connector = fleetShard.getConnector(deployment).orElseGet(() -> {
        LOGGER.info("Connector not found (cluster_id: {}, namespace_id: {}, connector_id: {}, deployment_id: {}, resource_version: {}), creating a new one", fleetShard.getClusterId(), deployment.getSpec().getNamespaceId(), deployment.getSpec().getConnectorId(), deployment.getId(), deployment.getMetadata().getResourceVersion());
        ManagedConnector answer = new ManagedConnector();
        answer.setMetadata(new ObjectMeta());
        answer.getMetadata().setNamespace(fleetShard.generateNamespaceId(deployment.getSpec().getNamespaceId()));
        answer.getMetadata().setName(Connectors.generateConnectorId(deployment.getId()));
        Resources.setLabels(answer, LABEL_CLUSTER_ID, fleetShard.getClusterId(), LABEL_CONNECTOR_ID, deployment.getSpec().getConnectorId(), LABEL_DEPLOYMENT_ID, deployment.getId());
        answer.getSpec().setClusterId(fleetShard.getClusterId());
        answer.getSpec().setConnectorId(deployment.getSpec().getConnectorId());
        answer.getSpec().setDeploymentId(deployment.getId());
        return answer;
    });
    // TODO: change APIs to include a single operator
    // move operator one level up
    // include full operator info in ConnectorDeployment APIs
    ArrayNode operatorsMeta = deployment.getSpec().getShardMetadata().withArray("operators");
    if (operatorsMeta.size() != 1) {
        throw new IllegalArgumentException("Multiple selectors are not yet supported");
    }
    OperatorSelector operatorSelector = new OperatorSelector(deployment.getSpec().getOperatorId(), operatorsMeta.get(0).requiredAt("/type").asText(), operatorsMeta.get(0).requiredAt("/version").asText());
    if (operatorSelector.getId() == null) {
        final OperatorSelector currentSelector = connector.getSpec().getOperatorSelector();
        // don't select a new operator if previously set.
        if (currentSelector != null && currentSelector.getId() != null) {
            operatorSelector.setId(currentSelector.getId());
        } else {
            Collection<Operator> operators = fleetShard.getOperators().stream().map(mco -> new Operator(mco.getMetadata().getName(), mco.getSpec().getType(), mco.getSpec().getVersion())).collect(Collectors.toList());
            OperatorSelectorUtil.assign(operatorSelector, operators).map(Operator::getId).ifPresent(operatorSelector::setId);
        }
    }
    if (operatorSelector.getId() != null) {
        Resources.setLabel(connector, LABEL_OPERATOR_ASSIGNED, operatorSelector.getId());
    }
    if (operatorSelector.getType() != null) {
        Resources.setLabel(connector, LABEL_OPERATOR_TYPE, operatorSelector.getType());
    }
    if (config != null) {
        config.connectors().labels().forEach((k, v) -> {
            Resources.setLabel(connector, k, v);
        });
        config.connectors().annotations().forEach((k, v) -> {
            Resources.setAnnotation(connector, k, v);
        });
    }
    Resources.setOwnerReferences(connector, owner);
    // add resource version to label
    Resources.setLabel(connector, LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + deployment.getMetadata().getResourceVersion());
    // add uow
    Resources.setLabel(connector, LABEL_UOW, uow);
    connector.getSpec().getDeployment().setDeploymentResourceVersion(deployment.getMetadata().getResourceVersion());
    connector.getSpec().getDeployment().setDesiredState(deployment.getSpec().getDesiredState().getValue());
    connector.getSpec().getDeployment().setConnectorTypeId(deployment.getSpec().getConnectorTypeId());
    connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
    KafkaConnectionSettings kafkaConnectionSettings = deployment.getSpec().getKafka();
    if (kafkaConnectionSettings != null) {
        connector.getSpec().getDeployment().setKafka(new KafkaSpec(kafkaConnectionSettings.getId(), kafkaConnectionSettings.getUrl()));
    }
    SchemaRegistryConnectionSettings schemaRegistryConnectionSettings = deployment.getSpec().getSchemaRegistry();
    if (schemaRegistryConnectionSettings != null) {
        connector.getSpec().getDeployment().setSchemaRegistry(new SchemaRegistrySpec(schemaRegistryConnectionSettings.getId(), schemaRegistryConnectionSettings.getUrl()));
    }
    connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
    connector.getSpec().getDeployment().setSecret(Secrets.generateConnectorSecretId(deployment.getId()));
    connector.getSpec().getDeployment().setUnitOfWork(uow);
    connector.getSpec().setOperatorSelector(operatorSelector);
    LOGGER.info("Provisioning connector namespace: {}, name: {}, revision: {}", connector.getMetadata().getNamespace(), connector.getMetadata().getName(), connector.getSpec().getDeployment().getDeploymentResourceVersion());
    try {
        return fleetShard.createConnector(connector);
    } catch (Exception e) {
        LOGGER.warn("", e);
        throw e;
    }
}
Also used : Operator(org.bf2.cos.fleetshard.api.Operator) Connectors(org.bf2.cos.fleetshard.support.resources.Connectors) Secrets(org.bf2.cos.fleetshard.support.resources.Secrets) LoggerFactory(org.slf4j.LoggerFactory) LABEL_DEPLOYMENT_RESOURCE_VERSION(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_DEPLOYMENT_RESOURCE_VERSION) KafkaSpec(org.bf2.cos.fleetshard.api.KafkaSpec) Operator(org.bf2.cos.fleetshard.api.Operator) Conditions(org.bf2.cos.fleetshard.api.Conditions) MetricsRecorder(org.bf2.cos.fleetshard.support.metrics.MetricsRecorder) OperatorSelector(org.bf2.cos.fleetshard.api.OperatorSelector) Resources.uid(org.bf2.cos.fleetshard.support.resources.Resources.uid) FleetShardClient(org.bf2.cos.fleetshard.sync.client.FleetShardClient) LABEL_CLUSTER_ID(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_CLUSTER_ID) ManagedConnector(org.bf2.cos.fleetshard.api.ManagedConnector) SchemaRegistrySpec(org.bf2.cos.fleetshard.api.SchemaRegistrySpec) ConnectorDeploymentStatus(org.bf2.cos.fleet.manager.model.ConnectorDeploymentStatus) LABEL_DEPLOYMENT_ID(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_DEPLOYMENT_ID) Tags(io.micrometer.core.instrument.Tags) MetaV1Condition(org.bf2.cos.fleet.manager.model.MetaV1Condition) Logger(org.slf4j.Logger) SchemaRegistryConnectionSettings(org.bf2.cos.fleet.manager.model.SchemaRegistryConnectionSettings) Collection(java.util.Collection) KafkaConnectionSettings(org.bf2.cos.fleet.manager.model.KafkaConnectionSettings) FleetShardSyncConfig(org.bf2.cos.fleetshard.sync.FleetShardSyncConfig) ConnectorDeployment(org.bf2.cos.fleet.manager.model.ConnectorDeployment) Collectors(java.util.stream.Collectors) HasMetadata(io.fabric8.kubernetes.api.model.HasMetadata) ArrayNode(com.fasterxml.jackson.databind.node.ArrayNode) LABEL_OPERATOR_TYPE(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_OPERATOR_TYPE) LABEL_UOW(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_UOW) OperatorSelectorUtil(org.bf2.cos.fleetshard.support.OperatorSelectorUtil) FleetManagerClient(org.bf2.cos.fleetshard.sync.client.FleetManagerClient) MeterRegistry(io.micrometer.core.instrument.MeterRegistry) LABEL_OPERATOR_ASSIGNED(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_OPERATOR_ASSIGNED) ObjectMeta(io.fabric8.kubernetes.api.model.ObjectMeta) Secret(io.fabric8.kubernetes.api.model.Secret) ApplicationScoped(javax.enterprise.context.ApplicationScoped) Resources(org.bf2.cos.fleetshard.support.resources.Resources) LABEL_CONNECTOR_ID(org.bf2.cos.fleetshard.support.resources.Resources.LABEL_CONNECTOR_ID) ObjectMeta(io.fabric8.kubernetes.api.model.ObjectMeta) ManagedConnector(org.bf2.cos.fleetshard.api.ManagedConnector) SchemaRegistrySpec(org.bf2.cos.fleetshard.api.SchemaRegistrySpec) KafkaSpec(org.bf2.cos.fleetshard.api.KafkaSpec) SchemaRegistryConnectionSettings(org.bf2.cos.fleet.manager.model.SchemaRegistryConnectionSettings) ArrayNode(com.fasterxml.jackson.databind.node.ArrayNode) OperatorSelector(org.bf2.cos.fleetshard.api.OperatorSelector) KafkaConnectionSettings(org.bf2.cos.fleet.manager.model.KafkaConnectionSettings)

Example 2 with KafkaSpec

use of org.bf2.cos.fleetshard.api.KafkaSpec in project kas-fleetshard by bf2fc6cc711aee1a0c2a.

the class KafkaCluster method buildKafkaStorage.

private Storage buildKafkaStorage(ManagedKafka managedKafka, Kafka current) {
    PersistentClaimStorageBuilder builder = new PersistentClaimStorageBuilder().withId(JBOD_VOLUME_ID).withSize(getAdjustedMaxDataRetentionSize(managedKafka, current).getAmount()).withDeleteClaim(DELETE_CLAIM);
    Optional.ofNullable(current).map(Kafka::getSpec).map(KafkaSpec::getKafka).map(KafkaClusterSpec::getStorage).map(this::getExistingVolumesFromJbodStorage).ifPresentOrElse(existingVolumes -> existingVolumes.stream().forEach(v -> handleExistingVolume(v, builder)), () -> builder.withStorageClass(config.getKafka().getStorageClass()));
    return new JbodStorageBuilder().withVolumes(builder.build()).build();
}
Also used : KafkaClusterSpec(io.strimzi.api.kafka.model.KafkaClusterSpec) Quantity(io.fabric8.kubernetes.api.model.Quantity) KafkaClusterSpec(io.strimzi.api.kafka.model.KafkaClusterSpec) PodAffinityTerm(io.fabric8.kubernetes.api.model.PodAffinityTerm) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) ImagePullSecretManager(org.bf2.operator.managers.ImagePullSecretManager) Storage(io.strimzi.api.kafka.model.storage.Storage) Rack(io.strimzi.api.kafka.model.Rack) StrimziManager(org.bf2.operator.managers.StrimziManager) Resource(io.fabric8.kubernetes.client.dsl.Resource) JvmOptionsBuilder(io.strimzi.api.kafka.model.JvmOptionsBuilder) PersistentClaimStorage(io.strimzi.api.kafka.model.storage.PersistentClaimStorage) ResourceRequirements(io.fabric8.kubernetes.api.model.ResourceRequirements) ConfigMapKeySelector(io.fabric8.kubernetes.api.model.ConfigMapKeySelector) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) KafkaClusterTemplate(io.strimzi.api.kafka.model.template.KafkaClusterTemplate) SystemProperty(io.strimzi.api.kafka.model.SystemProperty) Map(java.util.Map) KafkaExporterSpec(io.strimzi.api.kafka.model.KafkaExporterSpec) KafkaSpec(io.strimzi.api.kafka.model.KafkaSpec) KafkaManager(org.bf2.operator.managers.KafkaManager) Instance(javax.enterprise.inject.Instance) ExternalLogging(io.strimzi.api.kafka.model.ExternalLogging) ZookeeperClusterTemplateBuilder(io.strimzi.api.kafka.model.template.ZookeeperClusterTemplateBuilder) JvmOptions(io.strimzi.api.kafka.model.JvmOptions) JbodStorageBuilder(io.strimzi.api.kafka.model.storage.JbodStorageBuilder) OperandUtils(org.bf2.common.OperandUtils) DrainCleanerManager(org.bf2.operator.managers.DrainCleanerManager) Config(org.eclipse.microprofile.config.Config) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) PodAffinityTermBuilder(io.fabric8.kubernetes.api.model.PodAffinityTermBuilder) Startup(io.quarkus.runtime.Startup) List(java.util.List) RackBuilder(io.strimzi.api.kafka.model.RackBuilder) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) Optional(java.util.Optional) ApplicationScoped(javax.enterprise.context.ApplicationScoped) TopologySpreadConstraintBuilder(io.fabric8.kubernetes.api.model.TopologySpreadConstraintBuilder) ManagedKafka(org.bf2.operator.resources.v1alpha1.ManagedKafka) DatatypeConverter(javax.xml.bind.DatatypeConverter) ZookeeperClusterTemplate(io.strimzi.api.kafka.model.template.ZookeeperClusterTemplate) PersistentVolumeClaimStatus(io.fabric8.kubernetes.api.model.PersistentVolumeClaimStatus) Context(io.javaoperatorsdk.operator.api.Context) ZookeeperClusterSpec(io.strimzi.api.kafka.model.ZookeeperClusterSpec) ExternalLoggingBuilder(io.strimzi.api.kafka.model.ExternalLoggingBuilder) MessageDigest(java.security.MessageDigest) Logger(org.jboss.logging.Logger) ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder) KafkaAuthorizationCustomBuilder(io.strimzi.api.kafka.model.KafkaAuthorizationCustomBuilder) SingleVolumeStorage(io.strimzi.api.kafka.model.storage.SingleVolumeStorage) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) KafkaBuilder(io.strimzi.api.kafka.model.KafkaBuilder) Inject(javax.inject.Inject) PodDisruptionBudgetTemplateBuilder(io.strimzi.api.kafka.model.template.PodDisruptionBudgetTemplateBuilder) DefaultBean(io.quarkus.arc.DefaultBean) PersistentClaimStorageBuilder(io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder) PodAntiAffinity(io.fabric8.kubernetes.api.model.PodAntiAffinity) ExternalConfigurationReferenceBuilder(io.strimzi.api.kafka.model.ExternalConfigurationReferenceBuilder) JmxPrometheusExporterMetricsBuilder(io.strimzi.api.kafka.model.JmxPrometheusExporterMetricsBuilder) KafkaAuthorization(io.strimzi.api.kafka.model.KafkaAuthorization) KafkaClusterTemplateBuilder(io.strimzi.api.kafka.model.template.KafkaClusterTemplateBuilder) MetricsConfig(io.strimzi.api.kafka.model.MetricsConfig) SecuritySecretManager(org.bf2.operator.managers.SecuritySecretManager) PodAffinity(io.fabric8.kubernetes.api.model.PodAffinity) PodAntiAffinityBuilder(io.fabric8.kubernetes.api.model.PodAntiAffinityBuilder) Versions(org.bf2.operator.resources.v1alpha1.Versions) JbodStorage(io.strimzi.api.kafka.model.storage.JbodStorage) TopologySpreadConstraint(io.fabric8.kubernetes.api.model.TopologySpreadConstraint) IngressControllerManager(org.bf2.operator.managers.IngressControllerManager) Toleration(io.fabric8.kubernetes.api.model.Toleration) IOException(java.io.IOException) PodTemplateBuilder(io.strimzi.api.kafka.model.template.PodTemplateBuilder) ServiceAccount(org.bf2.operator.resources.v1alpha1.ServiceAccount) TolerationBuilder(io.fabric8.kubernetes.api.model.TolerationBuilder) ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap) AffinityBuilder(io.fabric8.kubernetes.api.model.AffinityBuilder) ConfigMapBuilder(io.fabric8.kubernetes.api.model.ConfigMapBuilder) ManagedKafkaAuthenticationOAuth(org.bf2.operator.resources.v1alpha1.ManagedKafkaAuthenticationOAuth) KafkaExporterSpecBuilder(io.strimzi.api.kafka.model.KafkaExporterSpecBuilder) AccessControl(org.bf2.operator.operands.KafkaInstanceConfiguration.AccessControl) SystemPropertyBuilder(io.strimzi.api.kafka.model.SystemPropertyBuilder) ConfigMapKeySelectorBuilder(io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder) DigestInputStream(java.security.DigestInputStream) PodNested(io.strimzi.api.kafka.model.template.ZookeeperClusterTemplateFluent.PodNested) ConfigProperty(org.eclipse.microprofile.config.inject.ConfigProperty) Kafka(io.strimzi.api.kafka.model.Kafka) Collections(java.util.Collections) InputStream(java.io.InputStream) JbodStorageBuilder(io.strimzi.api.kafka.model.storage.JbodStorageBuilder) ManagedKafka(org.bf2.operator.resources.v1alpha1.ManagedKafka) Kafka(io.strimzi.api.kafka.model.Kafka) PersistentClaimStorageBuilder(io.strimzi.api.kafka.model.storage.PersistentClaimStorageBuilder)

Aggregations

Collectors (java.util.stream.Collectors)2 ApplicationScoped (javax.enterprise.context.ApplicationScoped)2 ArrayNode (com.fasterxml.jackson.databind.node.ArrayNode)1 AffinityBuilder (io.fabric8.kubernetes.api.model.AffinityBuilder)1 ConfigMap (io.fabric8.kubernetes.api.model.ConfigMap)1 ConfigMapBuilder (io.fabric8.kubernetes.api.model.ConfigMapBuilder)1 ConfigMapKeySelector (io.fabric8.kubernetes.api.model.ConfigMapKeySelector)1 ConfigMapKeySelectorBuilder (io.fabric8.kubernetes.api.model.ConfigMapKeySelectorBuilder)1 HasMetadata (io.fabric8.kubernetes.api.model.HasMetadata)1 LabelSelector (io.fabric8.kubernetes.api.model.LabelSelector)1 ObjectMeta (io.fabric8.kubernetes.api.model.ObjectMeta)1 PersistentVolumeClaimStatus (io.fabric8.kubernetes.api.model.PersistentVolumeClaimStatus)1 PodAffinity (io.fabric8.kubernetes.api.model.PodAffinity)1 PodAffinityTerm (io.fabric8.kubernetes.api.model.PodAffinityTerm)1 PodAffinityTermBuilder (io.fabric8.kubernetes.api.model.PodAffinityTermBuilder)1 PodAntiAffinity (io.fabric8.kubernetes.api.model.PodAntiAffinity)1 PodAntiAffinityBuilder (io.fabric8.kubernetes.api.model.PodAntiAffinityBuilder)1 Quantity (io.fabric8.kubernetes.api.model.Quantity)1 ResourceRequirements (io.fabric8.kubernetes.api.model.ResourceRequirements)1 ResourceRequirementsBuilder (io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)1