Examples with ManagedConnector org.bf2.cos.fleetshard.api.ManagedConnector used on opensource projects

Example 26 with ManagedConnector

use of org.bf2.cos.fleetshard.api.ManagedConnector in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class CamelOperandSupport method createSteps.

public static List<KameletEndpoint> createSteps(ManagedConnector connector, ConnectorConfiguration<ObjectNode, ObjectNode> connectorConfiguration, CamelShardMetadata shardMetadata, KameletEndpoint kafkaEndpoint) {
    String consumes = Optional.ofNullable(connectorConfiguration.getDataShapeSpec()).map(spec -> spec.at("/consumes/format")).filter(node -> !node.isMissingNode()).map(JsonNode::asText).orElse(shardMetadata.getConsumes());
    String produces = Optional.ofNullable(connectorConfiguration.getDataShapeSpec()).map(spec -> spec.at("/produces/format")).filter(node -> !node.isMissingNode()).map(JsonNode::asText).orElse(shardMetadata.getProduces());
    final ArrayNode steps = connectorConfiguration.getProcessorsSpec();
    final List<KameletEndpoint> stepDefinitions = new ArrayList<>(steps.size() + 2);
    if (consumes != null) {
        switch(consumes) {
            case "application/json":
                stepDefinitions.add(kamelet("cos-decoder-json-action", properties -> {
                    if (shardMetadata.getConsumesClass() != null) {
                        properties.put("contentClass", shardMetadata.getConsumesClass());
                    }
                }));
                break;
            case "avro/binary":
                stepDefinitions.add(kamelet("cos-decoder-avro-action", properties -> {
                    if (shardMetadata.getConsumesClass() != null) {
                        properties.put("contentClass", shardMetadata.getConsumesClass());
                    }
                }));
                break;
            case "application/x-java-object":
                stepDefinitions.add(kamelet("cos-decoder-pojo-action", properties -> {
                    if (shardMetadata.getConsumesClass() != null) {
                        properties.put("mimeType", produces);
                    }
                }));
                break;
            case "text/plain":
            case "application/octet-stream":
                break;
            default:
                throw new IllegalArgumentException("Unsupported value format " + consumes);
        }
    }
    for (JsonNode step : steps) {
        var element = step.fields().next();
        String templateId = shardMetadata.getKamelets().getProcessors().get(element.getKey());
        if (templateId == null) {
            throw new IllegalArgumentException("Unknown processor: " + element.getKey());
        }
        stepDefinitions.add(configureStep(templateId, (ObjectNode) element.getValue()));
    }
    if (produces != null) {
        switch(produces) {
            case "application/json":
                stepDefinitions.add(kamelet("cos-encoder-json-action", properties -> {
                    if (shardMetadata.getProducesClass() != null) {
                        properties.put("contentClass", shardMetadata.getProducesClass());
                    }
                }));
                break;
            case "avro/binary":
                stepDefinitions.add(kamelet("cos-encoder-avro-action", properties -> {
                    if (shardMetadata.getProducesClass() != null) {
                        properties.put("contentClass", shardMetadata.getProducesClass());
                    }
                }));
                break;
            case "text/plain":
                stepDefinitions.add(kamelet("cos-encoder-string-action"));
                break;
            case "application/octet-stream":
                stepDefinitions.add(kamelet("cos-encoder-bytearray-action"));
                break;
            default:
                throw new IllegalArgumentException("Unsupported value format " + produces);
        }
    }
    // If it is a sink, then it consumes from kafka
    if (isSink(shardMetadata)) {
        String valueDeserializer = "org.bf2.cos.connector.camel.serdes.bytes.ByteArrayDeserializer";
        if ("application/json".equals(consumes) && hasSchemaRegistry(connector)) {
            valueDeserializer = "org.bf2.cos.connector.camel.serdes.json.JsonDeserializer";
        } else if ("avro/binary".equals(produces) && hasSchemaRegistry(connector)) {
            valueDeserializer = "org.bf2.cos.connector.camel.serdes.avro.AvroDeserializer";
        }
        kafkaEndpoint.getProperties().put("valueDeserializer", valueDeserializer);
    }
    // If it is a source, then it produces to kafka
    if (isSource(shardMetadata)) {
        String valueSerializer = "org.bf2.cos.connector.camel.serdes.bytes.ByteArraySerializer";
        if ("application/json".equals(produces) && hasSchemaRegistry(connector)) {
            valueSerializer = "org.bf2.cos.connector.camel.serdes.json.JsonSerializer";
        } else if ("avro/binary".equals(produces) && hasSchemaRegistry(connector)) {
            valueSerializer = "org.bf2.cos.connector.camel.serdes.avro.AvroSerializer";
        }
        kafkaEndpoint.getProperties().put("valueSerializer", valueSerializer);
    }
    return stepDefinitions;
}

Example 27 with ManagedConnector

use of org.bf2.cos.fleetshard.api.ManagedConnector in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class DebeziumOperandController method doReify.

@Override
protected List<HasMetadata> doReify(ManagedConnector connector, DebeziumShardMetadata shardMetadata, ConnectorConfiguration<ObjectNode, DebeziumDataShape> connectorConfiguration, ServiceAccountSpec serviceAccountSpec) {
    final Map<String, String> secretsData = createSecretsData(connectorConfiguration.getConnectorSpec());
    final Secret secret = new SecretBuilder().withMetadata(new ObjectMetaBuilder().withName(connector.getMetadata().getName() + Resources.CONNECTOR_SECRET_SUFFIX).build()).addToData(EXTERNAL_CONFIG_FILE, asBytesBase64(secretsData)).addToData(KAFKA_CLIENT_SECRET_KEY, serviceAccountSpec.getClientSecret()).build();
    ConfigMap kafkaConnectMetricsConfigMap = new ConfigMapBuilder().withNewMetadata().withName(connector.getMetadata().getName() + KAFKA_CONNECT_METRICS_CONFIGMAP_NAME_SUFFIX).endMetadata().addToData(METRICS_CONFIG_FILENAME, METRICS_CONFIG).build();
    final KafkaConnectSpecBuilder kcsb = new KafkaConnectSpecBuilder().withReplicas(1).withBootstrapServers(connector.getSpec().getDeployment().getKafka().getUrl()).withKafkaClientAuthenticationPlain(new KafkaClientAuthenticationPlainBuilder().withUsername(serviceAccountSpec.getClientId()).withPasswordSecret(new PasswordSecretSourceBuilder().withSecretName(secret.getMetadata().getName()).withPassword(KAFKA_CLIENT_SECRET_KEY).build()).build()).addToConfig(DebeziumConstants.DEFAULT_CONFIG_OPTIONS).addToConfig(new TreeMap<>(configuration.kafkaConnect().config())).addToConfig("group.id", connector.getMetadata().getName()).addToConfig(KeyAndValueConverters.getConfig(connectorConfiguration.getDataShapeSpec(), connector, serviceAccountSpec)).addToConfig("offset.storage.topic", connector.getMetadata().getName() + "-offset").addToConfig("config.storage.topic", connector.getMetadata().getName() + "-config").addToConfig("status.storage.topic", connector.getMetadata().getName() + "-status").addToConfig("topic.creation.enable", "true").addToConfig("connector.secret.name", secret.getMetadata().getName()).addToConfig("connector.secret.checksum", Secrets.computeChecksum(secret)).withTls(new ClientTlsBuilder().withTrustedCertificates(Collections.emptyList()).build()).withTemplate(new KafkaConnectTemplateBuilder().withPod(new PodTemplateBuilder().withImagePullSecrets(configuration.imagePullSecretsName()).build()).build()).withJmxPrometheusExporterMetricsConfig(new JmxPrometheusExporterMetricsBuilder().withValueFrom(new ExternalConfigurationReferenceBuilder().withNewConfigMapKeyRef(METRICS_CONFIG_FILENAME, kafkaConnectMetricsConfigMap.getMetadata().getName(), false).build()).build()).withExternalConfiguration(new ExternalConfigurationBuilder().addToVolumes(new ExternalConfigurationVolumeSourceBuilder().withName(EXTERNAL_CONFIG_DIRECTORY).withSecret(new SecretVolumeSourceBuilder().withSecretName(secret.getMetadata().getName()).build()).build()).build()).withResources(new ResourceRequirementsBuilder().addToRequests("cpu", new Quantity("10m")).addToRequests("memory", new Quantity("256Mi")).addToLimits("cpu", new Quantity("500m")).addToLimits("memory", new Quantity("1Gi")).build());
    kcsb.withImage(shardMetadata.getContainerImage());
    final KafkaConnect kc = new KafkaConnectBuilder().withApiVersion(Constants.RESOURCE_GROUP_NAME + "/" + KafkaConnect.CONSUMED_VERSION).withMetadata(new ObjectMetaBuilder().withName(connector.getMetadata().getName()).addToAnnotations(STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").build()).withSpec(kcsb.build()).build();
    Map<String, Object> connectorConfig = createConfig(configuration, connectorConfiguration.getConnectorSpec());
    // handle connector config defaults
    switch(shardMetadata.getConnectorClass()) {
        case CLASS_NAME_POSTGRES_CONNECTOR:
            if (!connectorConfig.containsKey(CONFIG_OPTION_POSTGRES_PLUGIN_NAME)) {
                connectorConfig.put(CONFIG_OPTION_POSTGRES_PLUGIN_NAME, PLUGIN_NAME_PGOUTPUT);
            }
            break;
        default:
            break;
    }
    if (isDatabaseHistorySupported(shardMetadata)) {
        final Map<String, Object> databaseHistoryConfigs = new LinkedHashMap<>();
        databaseHistoryConfigs.put("database.history.kafka.bootstrap.servers", connector.getSpec().getDeployment().getKafka().getUrl());
        databaseHistoryConfigs.put("database.history.kafka.topic", connector.getMetadata().getName() + "-database-history");
        databaseHistoryConfigs.put("database.history.producer.security.protocol", "SASL_SSL");
        databaseHistoryConfigs.put("database.history.consumer.security.protocol", "SASL_SSL");
        databaseHistoryConfigs.put("database.history.producer.sasl.mechanism", "PLAIN");
        databaseHistoryConfigs.put("database.history.consumer.sasl.mechanism", "PLAIN");
        databaseHistoryConfigs.put("database.history.producer.sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"" + serviceAccountSpec.getClientId() + "\" password=\"" + "${dir:/opt/kafka/external-configuration/" + EXTERNAL_CONFIG_DIRECTORY + ":" + KAFKA_CLIENT_SECRET_KEY + "}\";");
        databaseHistoryConfigs.put("database.history.consumer.sasl.jaas.config", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"" + serviceAccountSpec.getClientId() + "\" password=\"" + "${dir:/opt/kafka/external-configuration/" + EXTERNAL_CONFIG_DIRECTORY + ":" + KAFKA_CLIENT_SECRET_KEY + "}\";");
        connectorConfig.putAll(databaseHistoryConfigs);
    }
    final KafkaConnector kctr = new KafkaConnectorBuilder().withApiVersion(Constants.RESOURCE_GROUP_NAME + "/" + KafkaConnector.CONSUMED_VERSION).withMetadata(new ObjectMetaBuilder().withName(connector.getMetadata().getName()).addToLabels(STRIMZI_DOMAIN + "cluster", connector.getMetadata().getName()).build()).withSpec(new KafkaConnectorSpecBuilder().withClassName(shardMetadata.getConnectorClass()).withTasksMax(1).withPause(false).withConfig(connectorConfig).addToConfig("topic.creation.default.replication.factor", -1).addToConfig("topic.creation.default.partitions", -1).addToConfig("topic.creation.default.cleanup.policy", "compact").addToConfig("topic.creation.default.compression.type", "lz4").addToConfig("topic.creation.default.delete.retention.ms", 2_678_400_000L).build()).build();
    return List.of(secret, kafkaConnectMetricsConfigMap, kc, kctr);
}

Example 28 with ManagedConnector

use of org.bf2.cos.fleetshard.api.ManagedConnector in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorController method handleMonitor.

private UpdateControl<ManagedConnector> handleMonitor(ManagedConnector connector) {
    operandController.status(connector);
    // 
    // Search for newly installed ManagedOperators
    // 
    final List<Operator> operators = fleetShard.lookupOperators();
    final Operator assignedOperator = connector.getStatus().getConnectorStatus().getAssignedOperator();
    final Operator availableOperator = connector.getStatus().getConnectorStatus().getAvailableOperator();
    final Optional<Operator> selected = available(connector.getSpec().getOperatorSelector(), operators);
    if (selected.isPresent()) {
        Operator selectedInstance = selected.get();
        // if the selected operator does match the operator preciously selected
        if (!Objects.equals(selectedInstance, availableOperator) && !Objects.equals(selectedInstance, assignedOperator)) {
            // and it is not the currently assigned one
            LOGGER.info("deployment (upd): {} -> from:{}, to: {}", connector.getSpec().getDeployment(), assignedOperator, selectedInstance);
            // then we can signal that an upgrade is possible
            connector.getStatus().getConnectorStatus().setAvailableOperator(selectedInstance);
        }
    } else {
        connector.getStatus().getConnectorStatus().setAvailableOperator(new Operator());
    }
    return UpdateControl.updateStatus(connector);
}

Example 29 with ManagedConnector

use of org.bf2.cos.fleetshard.api.ManagedConnector in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorStatusExtractor method extract.

public static ConnectorDeploymentStatus extract(ManagedConnector connector) {
    ConnectorDeploymentStatus status = new ConnectorDeploymentStatus();
    DeploymentSpec deployment = connector.getSpec().getDeployment();
    if (connector.getStatus() != null && connector.getStatus().getPhase() != null) {
        deployment = connector.getStatus().getDeployment();
    }
    status.setResourceVersion(deployment.getDeploymentResourceVersion());
    if (connector.getSpec().getOperatorSelector() == null || connector.getSpec().getOperatorSelector().getId() == null) {
        status.setPhase(STATE_FAILED);
        status.addConditionsItem(new MetaV1Condition().type(Conditions.TYPE_READY).status(Conditions.STATUS_FALSE).message("No assignable operator").reason(Conditions.NO_ASSIGNABLE_OPERATOR_REASON).lastTransitionTime(Conditions.now()));
        return status;
    }
    if (connector.getStatus() != null && connector.getStatus().getConnectorStatus() != null) {
        status.setOperators(new ConnectorDeploymentStatusOperators().assigned(toConnectorOperator(connector.getStatus().getConnectorStatus().getAssignedOperator())).available(toConnectorOperator(connector.getStatus().getConnectorStatus().getAvailableOperator())));
        if (connector.getStatus().getConnectorStatus() != null) {
            if (connector.getStatus().getConnectorStatus().getPhase() != null) {
                status.setPhase(connector.getStatus().getConnectorStatus().getPhase());
            }
            if (connector.getStatus().getConnectorStatus().getConditions() != null) {
                for (var cond : connector.getStatus().getConnectorStatus().getConditions()) {
                    status.addConditionsItem(toMetaV1Condition(cond));
                }
            }
        }
    }
    if (status.getPhase() == null) {
        if (DESIRED_STATE_DELETED.equals(deployment.getDesiredState())) {
            status.setPhase(STATE_DE_PROVISIONING);
        } else if (DESIRED_STATE_STOPPED.equals(deployment.getDesiredState())) {
            status.setPhase(STATE_DE_PROVISIONING);
        } else {
            status.setPhase(STATE_PROVISIONING);
        }
    }
    return status;
}

Example 30 with ManagedConnector

use of org.bf2.cos.fleetshard.api.ManagedConnector in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorDeploymentProvisioner method createManagedConnector.

private ManagedConnector createManagedConnector(String uow, ConnectorDeployment deployment, HasMetadata owner) {
    ManagedConnector connector = fleetShard.getConnector(deployment).orElseGet(() -> {
        LOGGER.info("Connector not found (cluster_id: {}, connector_id: {}, deployment_id: {}, resource_version: {}), creating a new one", fleetShard.getClusterId(), deployment.getSpec().getConnectorId(), deployment.getId(), deployment.getMetadata().getResourceVersion());
        return Connectors.newConnector(fleetShard.getClusterId(), deployment.getSpec().getConnectorId(), deployment.getId(), Map.of());
    });
    // TODO: change APIs to include a single operator
    // move operator one level up
    // include full operator info in ConnectorDeployment APIs
    ArrayNode operatorsMeta = deployment.getSpec().getShardMetadata().withArray("operators");
    if (operatorsMeta.size() != 1) {
        throw new IllegalArgumentException("Multiple selectors are not yet supported");
    }
    OperatorSelector operatorSelector = new OperatorSelector(deployment.getSpec().getOperatorId(), operatorsMeta.get(0).requiredAt("/type").asText(), operatorsMeta.get(0).requiredAt("/version").asText());
    if (operatorSelector.getId() == null) {
        final OperatorSelector currentSelector = connector.getSpec().getOperatorSelector();
        // don't select a new operator if previously set.
        if (currentSelector != null && currentSelector.getId() != null) {
            operatorSelector.setId(currentSelector.getId());
        } else {
            OperatorSelectorUtil.assign(operatorSelector, fleetShard.lookupOperators()).map(Operator::getId).ifPresent(operatorSelector::setId);
        }
    }
    if (operatorSelector.getId() != null) {
        Resources.setLabel(connector, Resources.LABEL_OPERATOR_ASSIGNED, operatorSelector.getId());
    }
    if (operatorSelector.getType() != null) {
        Resources.setLabel(connector, Resources.LABEL_OPERATOR_TYPE, operatorSelector.getType());
    }
    if (config != null) {
        config.connectors().labels().forEach((k, v) -> {
            Resources.setLabel(connector, k, v);
        });
        config.connectors().annotations().forEach((k, v) -> {
            Resources.setAnnotation(connector, k, v);
        });
    }
    connector.getMetadata().setOwnerReferences(List.of(new OwnerReferenceBuilder().withApiVersion(owner.getApiVersion()).withKind(owner.getKind()).withName(owner.getMetadata().getName()).withUid(owner.getMetadata().getUid()).withBlockOwnerDeletion(true).build()));
    // add resource version to label
    Resources.setLabel(connector, Resources.LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + deployment.getMetadata().getResourceVersion());
    // add uow
    Resources.setLabel(connector, Resources.LABEL_UOW, uow);
    connector.getSpec().getDeployment().setDeploymentResourceVersion(deployment.getMetadata().getResourceVersion());
    connector.getSpec().getDeployment().setDesiredState(deployment.getSpec().getDesiredState());
    connector.getSpec().getDeployment().setConnectorTypeId(deployment.getSpec().getConnectorTypeId());
    connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
    KafkaConnectionSettings kafkaConnectionSettings = deployment.getSpec().getKafka();
    if (kafkaConnectionSettings != null) {
        connector.getSpec().getDeployment().setKafka(new KafkaSpecBuilder().withId(kafkaConnectionSettings.getId()).withUrl(kafkaConnectionSettings.getUrl()).build());
    }
    SchemaRegistryConnectionSettings schemaRegistryConnectionSettings = deployment.getSpec().getSchemaRegistry();
    if (schemaRegistryConnectionSettings != null) {
        connector.getSpec().getDeployment().setSchemaRegistry(new SchemaRegistrySpecBuilder().withId(schemaRegistryConnectionSettings.getId()).withUrl(schemaRegistryConnectionSettings.getUrl()).build());
    }
    connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
    connector.getSpec().getDeployment().setSecret(Secrets.generateConnectorSecretId(deployment.getId()));
    connector.getSpec().getDeployment().setUnitOfWork(uow);
    connector.getSpec().setOperatorSelector(operatorSelector);
    LOGGER.info("Provisioning connector id={} rv={} - {}/{}: {}", connector.getMetadata().getName(), connector.getSpec().getDeployment().getDeploymentResourceVersion(), fleetShard.getConnectorsNamespace(), connector.getSpec().getConnectorId(), Serialization.asJson(connector.getSpec()));
    try {
        return fleetShard.createConnector(connector);
    } catch (Exception e) {
        LOGGER.warn("", e);
        throw e;
    }
}