Examples with KafkaSpecBuilder org.bf2.cos.fleetshard.api.KafkaSpecBuilder used on opensource projects

Example 1 with KafkaSpecBuilder

use of org.bf2.cos.fleetshard.api.KafkaSpecBuilder in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorSteps method a_connector.

@Given("^a Connector with:$")
public void a_connector(Map<String, String> entry) {
    final Long drv = Long.parseLong(entry.getOrDefault(ConnectorContext.COS_DEPLOYMENT_RESOURCE_VERSION, "1"));
    final Long crv = Long.parseLong(entry.getOrDefault(ConnectorContext.COS_CONNECTOR_RESOURCE_VERSION, "1"));
    final String connectorId = entry.getOrDefault(ConnectorContext.COS_CONNECTOR_ID, uid());
    final String deploymentId = entry.getOrDefault(ConnectorContext.COS_DEPLOYMENT_ID, uid());
    final String clusterId = ctx.clusterId();
    var connector = new ManagedConnectorBuilder().withMetadata(new ObjectMetaBuilder().addToLabels(Resources.LABEL_CLUSTER_ID, clusterId).addToLabels(Resources.LABEL_CONNECTOR_ID, connectorId).addToLabels(Resources.LABEL_DEPLOYMENT_ID, deploymentId).addToLabels(Resources.LABEL_OPERATOR_TYPE, entry.get(ConnectorContext.OPERATOR_TYPE)).withName(Connectors.generateConnectorId(deploymentId)).build()).withSpec(new ManagedConnectorSpecBuilder().withClusterId(clusterId).withConnectorId(connectorId).withDeploymentId(deploymentId).withDeployment(new DeploymentSpecBuilder().withConnectorResourceVersion(crv).withConnectorTypeId(entry.get(ConnectorContext.CONNECTOR_TYPE_ID)).withDeploymentResourceVersion(drv).withKafka(new KafkaSpecBuilder().withUrl(entry.getOrDefault("kafka.bootstrap", "kafka.acme.com:443")).build()).withDesiredState(entry.get(ConnectorContext.DESIRED_STATE)).withSecret(Connectors.generateConnectorId(deploymentId) + "-" + drv).build()).withOperatorSelector(new OperatorSelectorBuilder().withId(entry.get(ConnectorContext.OPERATOR_ID)).withType(entry.get(ConnectorContext.OPERATOR_TYPE)).withVersion(entry.get(ConnectorContext.OPERATOR_VERSION)).build()).build()).build();
    var secret = new SecretBuilder().withMetadata(new ObjectMetaBuilder().addToLabels(Resources.LABEL_OPERATOR_TYPE, entry.get(ConnectorContext.OPERATOR_TYPE)).withName(connector.getMetadata().getName() + "-" + connector.getSpec().getDeployment().getDeploymentResourceVersion()).build()).withData(new HashMap<>()).addToData(Secrets.SECRET_ENTRY_SERVICE_ACCOUNT, Secrets.toBase64(Serialization.asJson(Serialization.jsonMapper().createObjectNode().put("client_id", entry.getOrDefault("kafka.client.id", uid())).put("client_secret", entry.getOrDefault("kafka.client.secret", Secrets.toBase64(uid())))))).build();
    ctx.connector(connector);
    ctx.secret(secret);
}

Example 2 with KafkaSpecBuilder

use of org.bf2.cos.fleetshard.api.KafkaSpecBuilder in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class DebeziumOperandControllerTest method reify.

@Test
void reify() {
    KubernetesClient kubernetesClient = Mockito.mock(KubernetesClient.class);
    DebeziumOperandController controller = new DebeziumOperandController(kubernetesClient, CONFIGURATION);
    final String kcsB64 = Base64.getEncoder().encodeToString("kcs".getBytes(StandardCharsets.UTF_8));
    final String pwdB64 = Base64.getEncoder().encodeToString("orderpw".getBytes(StandardCharsets.UTF_8));
    var spec = Serialization.jsonMapper().createObjectNode().put("database.hostname", "orderdb").put("database.port", "5432").put("database.user", "orderuser").put("database.dbname", "orderdb").put("database.server.name", "dbserver1").put("schema.include.list", "purchaseorder").put("table.include.list", "purchaseorder.outboxevent").put("tombstones.on.delete", "false").put("key.converter", "org.apache.kafka.connect.storage.StringConverter").put("value.converter", "org.apache.kafka.connect.storage.StringConverter").put("transforms", "saga").put("transforms.saga.type", "io.debezium.transforms.outbox.EventRouter").put("transforms.saga.route.topic.replacement", "${routedByValue}.request").put("poll.interval.ms", "100").put("consumer.interceptor.classes", "io.opentracing.contrib.kafka.TracingConsumerInterceptor").put("producer.interceptor.classes", "io.opentracing.contrib.kafka.TracingProducerInterceptor");
    spec.with("data_shape").put("key", "JSON").put("value", "JSON");
    spec.with("database.password").put("kind", "base64").put("value", pwdB64);
    var resources = controller.doReify(new ManagedConnectorBuilder().withMetadata(new ObjectMetaBuilder().withName(DEFAULT_MANAGED_CONNECTOR_ID).build()).withSpec(new ManagedConnectorSpecBuilder().withConnectorId(DEFAULT_MANAGED_CONNECTOR_ID).withDeploymentId(DEFAULT_DEPLOYMENT_ID).withDeployment(new DeploymentSpecBuilder().withConnectorTypeId(DEFAULT_CONNECTOR_TYPE_ID).withSecret("secret").withKafka(new KafkaSpecBuilder().withUrl(DEFAULT_KAFKA_SERVER).build()).withConnectorResourceVersion(DEFAULT_CONNECTOR_REVISION).withDeploymentResourceVersion(DEFAULT_DEPLOYMENT_REVISION).withDesiredState(DESIRED_STATE_READY).build()).build()).build(), new org.bf2.cos.fleetshard.operator.debezium.DebeziumShardMetadataBuilder().withContainerImage(DEFAULT_CONNECTOR_IMAGE).withConnectorClass(PG_CLASS).build(), new ConnectorConfiguration<>(spec, ObjectNode.class), new ServiceAccountSpecBuilder().withClientId(DEFAULT_KAFKA_CLIENT_ID).withClientSecret(kcsB64).build());
    assertThat(resources).anyMatch(DebeziumOperandSupport::isKafkaConnect).anyMatch(DebeziumOperandSupport::isKafkaConnector).anyMatch(DebeziumOperandSupport::isSecret);
    assertThat(resources).filteredOn(DebeziumOperandSupport::isKafkaConnect).hasSize(1).first().isInstanceOfSatisfying(KafkaConnect.class, kc -> {
        assertThat(kc.getSpec().getImage()).isEqualTo(DEFAULT_CONNECTOR_IMAGE);
    });
    assertThat(resources).filteredOn(DebeziumOperandSupport::isKafkaConnector).hasSize(1).first().isInstanceOfSatisfying(KafkaConnector.class, kc -> {
        assertThat(kc.getSpec().getConfig()).containsEntry("database.password", "${file:/opt/kafka/external-configuration/" + DebeziumConstants.EXTERNAL_CONFIG_DIRECTORY + "/" + EXTERNAL_CONFIG_FILE + ":database.password}");
    });
}

Example 3 with KafkaSpecBuilder

use of org.bf2.cos.fleetshard.api.KafkaSpecBuilder in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorDeploymentProvisioner method createManagedConnector.

private ManagedConnector createManagedConnector(String uow, ConnectorDeployment deployment, HasMetadata owner) {
    ManagedConnector connector = fleetShard.getConnector(deployment).orElseGet(() -> {
        LOGGER.info("Connector not found (cluster_id: {}, connector_id: {}, deployment_id: {}, resource_version: {}), creating a new one", fleetShard.getClusterId(), deployment.getSpec().getConnectorId(), deployment.getId(), deployment.getMetadata().getResourceVersion());
        return Connectors.newConnector(fleetShard.getClusterId(), deployment.getSpec().getConnectorId(), deployment.getId(), Map.of());
    });
    // TODO: change APIs to include a single operator
    // move operator one level up
    // include full operator info in ConnectorDeployment APIs
    ArrayNode operatorsMeta = deployment.getSpec().getShardMetadata().withArray("operators");
    if (operatorsMeta.size() != 1) {
        throw new IllegalArgumentException("Multiple selectors are not yet supported");
    }
    OperatorSelector operatorSelector = new OperatorSelector(deployment.getSpec().getOperatorId(), operatorsMeta.get(0).requiredAt("/type").asText(), operatorsMeta.get(0).requiredAt("/version").asText());
    if (operatorSelector.getId() == null) {
        final OperatorSelector currentSelector = connector.getSpec().getOperatorSelector();
        // don't select a new operator if previously set.
        if (currentSelector != null && currentSelector.getId() != null) {
            operatorSelector.setId(currentSelector.getId());
        } else {
            OperatorSelectorUtil.assign(operatorSelector, fleetShard.lookupOperators()).map(Operator::getId).ifPresent(operatorSelector::setId);
        }
    }
    if (operatorSelector.getId() != null) {
        Resources.setLabel(connector, Resources.LABEL_OPERATOR_ASSIGNED, operatorSelector.getId());
    }
    if (operatorSelector.getType() != null) {
        Resources.setLabel(connector, Resources.LABEL_OPERATOR_TYPE, operatorSelector.getType());
    }
    if (config != null) {
        config.connectors().labels().forEach((k, v) -> {
            Resources.setLabel(connector, k, v);
        });
        config.connectors().annotations().forEach((k, v) -> {
            Resources.setAnnotation(connector, k, v);
        });
    }
    connector.getMetadata().setOwnerReferences(List.of(new OwnerReferenceBuilder().withApiVersion(owner.getApiVersion()).withKind(owner.getKind()).withName(owner.getMetadata().getName()).withUid(owner.getMetadata().getUid()).withBlockOwnerDeletion(true).build()));
    // add resource version to label
    Resources.setLabel(connector, Resources.LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + deployment.getMetadata().getResourceVersion());
    // add uow
    Resources.setLabel(connector, Resources.LABEL_UOW, uow);
    connector.getSpec().getDeployment().setDeploymentResourceVersion(deployment.getMetadata().getResourceVersion());
    connector.getSpec().getDeployment().setDesiredState(deployment.getSpec().getDesiredState());
    connector.getSpec().getDeployment().setConnectorTypeId(deployment.getSpec().getConnectorTypeId());
    connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
    KafkaConnectionSettings kafkaConnectionSettings = deployment.getSpec().getKafka();
    if (kafkaConnectionSettings != null) {
        connector.getSpec().getDeployment().setKafka(new KafkaSpecBuilder().withId(kafkaConnectionSettings.getId()).withUrl(kafkaConnectionSettings.getUrl()).build());
    }
    SchemaRegistryConnectionSettings schemaRegistryConnectionSettings = deployment.getSpec().getSchemaRegistry();
    if (schemaRegistryConnectionSettings != null) {
        connector.getSpec().getDeployment().setSchemaRegistry(new SchemaRegistrySpecBuilder().withId(schemaRegistryConnectionSettings.getId()).withUrl(schemaRegistryConnectionSettings.getUrl()).build());
    }
    connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
    connector.getSpec().getDeployment().setSecret(Secrets.generateConnectorSecretId(deployment.getId()));
    connector.getSpec().getDeployment().setUnitOfWork(uow);
    connector.getSpec().setOperatorSelector(operatorSelector);
    LOGGER.info("Provisioning connector id={} rv={} - {}/{}: {}", connector.getMetadata().getName(), connector.getSpec().getDeployment().getDeploymentResourceVersion(), fleetShard.getConnectorsNamespace(), connector.getSpec().getConnectorId(), Serialization.asJson(connector.getSpec()));
    try {
        return fleetShard.createConnector(connector);
    } catch (Exception e) {
        LOGGER.warn("", e);
        throw e;
    }
}