Examples with Connectors org.bf2.cos.fleetshard.support.resources.Connectors used on opensource projects

Example 1 with Connectors

use of org.bf2.cos.fleetshard.support.resources.Connectors in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class FleetManagerClient method getDeployments.

public void getDeployments(long gv, Consumer<Collection<ConnectorDeployment>> consumer) {
    RestClientHelper.run(() -> {
        LOGGER.debug("polling deployment with gv: {}", gv);
        final AtomicInteger counter = new AtomicInteger();
        final List<ConnectorDeployment> items = new ArrayList<>();
        for (int i = 1; i < Integer.MAX_VALUE; i++) {
            ConnectorDeploymentList list = controlPlane.getConnectorDeployments(config.cluster().id(), Integer.toString(i), null, gv);
            if (list == null || list.getItems() == null || list.getItems().isEmpty()) {
                LOGGER.info("No connectors for cluster {}", config.cluster().id());
                break;
            }
            items.clear();
            items.addAll(list.getItems());
            items.sort(Comparator.comparingLong(d -> d.getMetadata().getResourceVersion()));
            consumer.accept(items);
            if (counter.addAndGet(items.size()) >= list.getTotal()) {
                break;
            }
        }
    });
}

Example 2 with Connectors

use of org.bf2.cos.fleetshard.support.resources.Connectors in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorProvisionerTest method updateResources.

@Test
void updateResources() {
    // 
    // Given that the resources associated to the provided deployment exist
    // 
    final ConnectorDeployment oldDeployment = createDeployment(0);
    final List<ManagedConnector> connectors = List.of(new ManagedConnectorBuilder().withMetadata(new ObjectMetaBuilder().withName(Connectors.generateConnectorId(oldDeployment.getId())).addToLabels(LABEL_CLUSTER_ID, CLUSTER_ID).addToLabels(LABEL_CONNECTOR_ID, oldDeployment.getSpec().getConnectorId()).addToLabels(LABEL_DEPLOYMENT_ID, oldDeployment.getId()).build()).build());
    final List<Secret> secrets = List.of(new SecretBuilder().withMetadata(new ObjectMetaBuilder().withName(Secrets.generateConnectorSecretId(oldDeployment.getId())).addToLabels(LABEL_CLUSTER_ID, CLUSTER_ID).addToLabels(LABEL_CONNECTOR_ID, oldDeployment.getSpec().getConnectorId()).addToLabels(LABEL_DEPLOYMENT_ID, oldDeployment.getId()).addToLabels(LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + oldDeployment.getMetadata().getResourceVersion()).build()).build());
    final FleetShardClient fleetShard = ConnectorTestSupport.fleetShard(CLUSTER_ID, connectors, secrets);
    final FleetManagerClient fleetManager = ConnectorTestSupport.fleetManagerClient();
    final FleetShardSyncConfig config = ConnectorTestSupport.config();
    final MeterRegistry registry = Mockito.mock(MeterRegistry.class);
    final ConnectorDeploymentProvisioner provisioner = new ConnectorDeploymentProvisioner(config, fleetShard, fleetManager, registry);
    final ArgumentCaptor<Secret> sc = ArgumentCaptor.forClass(Secret.class);
    final ArgumentCaptor<ManagedConnector> mcc = ArgumentCaptor.forClass(ManagedConnector.class);
    // 
    // When deployment is updated
    // 
    final ConnectorDeployment newDeployment = createDeployment(0, d -> {
        d.getSpec().getKafka().setUrl("my-kafka.acme.com:218");
        ((ObjectNode) d.getSpec().getConnectorSpec()).with("connector").put("foo", "connector-baz");
        ((ObjectNode) d.getSpec().getShardMetadata()).put("connector_image", "quay.io/mcs_dev/aws-s3-sink:0.1.0");
    });
    provisioner.provision(newDeployment);
    verify(fleetShard).createSecret(sc.capture());
    verify(fleetShard).createConnector(mcc.capture());
    // 
    // Then the existing resources must be updated to reflect the changes made to the
    // deployment. This scenario could happen when a resource on the connector cluster
    // is amended outside the control of fleet manager (i.e. with kubectl) and in such
    // case, the expected behavior is that the resource is re-set to the configuration
    // from the fleet manager.
    // 
    assertThat(sc.getValue()).satisfies(val -> {
        assertThat(val.getMetadata().getName()).isEqualTo(Secrets.generateConnectorSecretId(oldDeployment.getId()));
        assertThat(val.getMetadata().getLabels()).containsEntry(LABEL_CLUSTER_ID, CLUSTER_ID).containsEntry(LABEL_CONNECTOR_ID, newDeployment.getSpec().getConnectorId()).containsEntry(LABEL_DEPLOYMENT_ID, newDeployment.getId()).containsEntry(LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + newDeployment.getMetadata().getResourceVersion()).containsKey(LABEL_UOW);
        assertThat(val.getData()).containsKey(Secrets.SECRET_ENTRY_SERVICE_ACCOUNT).containsKey(Secrets.SECRET_ENTRY_CONNECTOR);
        var serviceAccountNode = Secrets.extract(val, Secrets.SECRET_ENTRY_SERVICE_ACCOUNT, ServiceAccount.class);
        assertThat(serviceAccountNode.getClientSecret()).isEqualTo(newDeployment.getSpec().getServiceAccount().getClientSecret());
        assertThat(serviceAccountNode.getClientId()).isEqualTo(newDeployment.getSpec().getServiceAccount().getClientId());
        var connectorNode = Secrets.extract(val, Secrets.SECRET_ENTRY_CONNECTOR);
        assertThatJson(Secrets.extract(val, Secrets.SECRET_ENTRY_CONNECTOR)).inPath("connector.foo").isEqualTo("connector-baz");
        assertThatJson(connectorNode).inPath("kafka.topic").isEqualTo("kafka-foo");
        var metaNode = Secrets.extract(val, Secrets.SECRET_ENTRY_META);
        assertThatJson(metaNode).isObject().containsKey("connector_type").containsKey("connector_image").containsKey("kamelets").containsKey("operators");
    });
    assertThat(mcc.getValue()).satisfies(val -> {
        assertThat(val.getMetadata().getName()).isEqualTo(Connectors.generateConnectorId(oldDeployment.getId()));
        assertThat(val.getMetadata().getLabels()).containsEntry(LABEL_CLUSTER_ID, CLUSTER_ID).containsEntry(LABEL_CONNECTOR_ID, oldDeployment.getSpec().getConnectorId()).containsEntry(LABEL_DEPLOYMENT_ID, oldDeployment.getId()).containsKey(LABEL_UOW);
        assertThat(val.getSpec().getDeployment()).satisfies(d -> {
            assertThat(d.getDeploymentResourceVersion()).isEqualTo(oldDeployment.getMetadata().getResourceVersion());
            assertThat(d.getDeploymentResourceVersion()).isEqualTo(newDeployment.getMetadata().getResourceVersion());
            assertThat(d.getSecret()).isEqualTo(sc.getValue().getMetadata().getName());
            assertThat(d.getUnitOfWork()).isNotEmpty().isEqualTo(sc.getValue().getMetadata().getLabels().get(LABEL_UOW));
            assertThat(d.getKafka().getUrl()).isNotEmpty().isEqualTo(newDeployment.getSpec().getKafka().getUrl());
        });
    });
}

Example 3 with Connectors

use of org.bf2.cos.fleetshard.support.resources.Connectors in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorProvisionerTest method updateAndCreateResources.

@Test
void updateAndCreateResources() {
    // 
    // Given that the resources associated to the provided deployment exist
    // 
    final ConnectorDeployment oldDeployment = createDeployment(0);
    final List<ManagedConnector> connectors = List.of(new ManagedConnectorBuilder().withMetadata(new ObjectMetaBuilder().withName(Connectors.generateConnectorId(oldDeployment.getId())).addToLabels(LABEL_CLUSTER_ID, CLUSTER_ID).addToLabels(LABEL_CONNECTOR_ID, oldDeployment.getSpec().getConnectorId()).addToLabels(LABEL_DEPLOYMENT_ID, oldDeployment.getId()).build()).build());
    final List<Secret> secrets = List.of(new SecretBuilder().withMetadata(new ObjectMetaBuilder().withName(Secrets.generateConnectorSecretId(oldDeployment.getId())).addToLabels(LABEL_CLUSTER_ID, CLUSTER_ID).addToLabels(LABEL_CONNECTOR_ID, oldDeployment.getSpec().getConnectorId()).addToLabels(LABEL_DEPLOYMENT_ID, oldDeployment.getId()).addToLabels(LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + oldDeployment.getMetadata().getResourceVersion()).build()).build());
    final FleetShardClient fleetShard = ConnectorTestSupport.fleetShard(CLUSTER_ID, connectors, secrets);
    final FleetManagerClient fleetManager = ConnectorTestSupport.fleetManagerClient();
    final FleetShardSyncConfig config = ConnectorTestSupport.config();
    final MeterRegistry registry = Mockito.mock(MeterRegistry.class);
    final ConnectorDeploymentProvisioner provisioner = new ConnectorDeploymentProvisioner(config, fleetShard, fleetManager, registry);
    final ArgumentCaptor<Secret> sc = ArgumentCaptor.forClass(Secret.class);
    final ArgumentCaptor<ManagedConnector> mcc = ArgumentCaptor.forClass(ManagedConnector.class);
    // 
    // When a change to the deployment happen that ends up with a new resource version
    // 
    final ConnectorDeployment newDeployment = createDeployment(1, d -> {
        d.getMetadata().setResourceVersion(1L);
        d.getSpec().getKafka().setUrl("my-kafka.acme.com:218");
        ((ObjectNode) d.getSpec().getConnectorSpec()).with("connector").put("foo", "connector-baz");
        ((ObjectNode) d.getSpec().getShardMetadata()).put("connector_image", "quay.io/mcs_dev/aws-s3-sink:0.1.0");
    });
    provisioner.provision(newDeployment);
    verify(fleetShard).createSecret(sc.capture());
    verify(fleetShard).createConnector(mcc.capture());
    // 
    // Then the managed connector resource is expected to be updated to reflect the
    // changes made to the deployment
    // 
    assertThat(sc.getValue()).satisfies(val -> {
        assertThat(val.getMetadata().getName()).isEqualTo(Secrets.generateConnectorSecretId(oldDeployment.getId()));
        assertThat(val.getMetadata().getLabels()).containsEntry(LABEL_CLUSTER_ID, CLUSTER_ID).containsEntry(LABEL_CONNECTOR_ID, newDeployment.getSpec().getConnectorId()).containsEntry(LABEL_DEPLOYMENT_ID, newDeployment.getId()).containsEntry(LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + newDeployment.getMetadata().getResourceVersion()).containsKey(LABEL_UOW);
        assertThat(val.getData()).containsKey(Secrets.SECRET_ENTRY_SERVICE_ACCOUNT).containsKey(Secrets.SECRET_ENTRY_CONNECTOR);
        var serviceAccountNode = Secrets.extract(val, Secrets.SECRET_ENTRY_SERVICE_ACCOUNT, ServiceAccount.class);
        assertThat(serviceAccountNode.getClientSecret()).isEqualTo(newDeployment.getSpec().getServiceAccount().getClientSecret());
        assertThat(serviceAccountNode.getClientId()).isEqualTo(newDeployment.getSpec().getServiceAccount().getClientId());
        var connectorNode = Secrets.extract(val, Secrets.SECRET_ENTRY_CONNECTOR);
        assertThatJson(Secrets.extract(val, Secrets.SECRET_ENTRY_CONNECTOR)).inPath("connector.foo").isEqualTo("connector-baz");
        assertThatJson(connectorNode).inPath("kafka.topic").isEqualTo("kafka-foo");
        var metaNode = Secrets.extract(val, Secrets.SECRET_ENTRY_META);
        assertThatJson(metaNode).isObject().containsKey("connector_type").containsKey("connector_image").containsKey("kamelets").containsKey("operators");
    });
    assertThat(mcc.getValue()).satisfies(val -> {
        assertThat(val.getMetadata().getName()).isEqualTo(Connectors.generateConnectorId(oldDeployment.getId()));
        assertThat(val.getMetadata().getLabels()).containsEntry(LABEL_CLUSTER_ID, CLUSTER_ID).containsEntry(LABEL_CONNECTOR_ID, oldDeployment.getSpec().getConnectorId()).containsEntry(LABEL_DEPLOYMENT_ID, oldDeployment.getId()).containsKey(LABEL_UOW);
        assertThat(val.getSpec().getDeployment()).satisfies(d -> {
            assertThat(d.getDeploymentResourceVersion()).isEqualTo(newDeployment.getMetadata().getResourceVersion());
            assertThat(d.getDeploymentResourceVersion()).isNotEqualTo(oldDeployment.getMetadata().getResourceVersion());
            assertThat(d.getSecret()).isEqualTo(sc.getValue().getMetadata().getName());
            assertThat(d.getUnitOfWork()).isNotEmpty().isEqualTo(sc.getValue().getMetadata().getLabels().get(LABEL_UOW));
            assertThat(d.getKafka().getUrl()).isNotEmpty().isEqualTo(newDeployment.getSpec().getKafka().getUrl());
        });
    });
}

Example 4 with Connectors

use of org.bf2.cos.fleetshard.support.resources.Connectors in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorProvisionerTest method createResources.

@Test
void createResources() {
    // 
    // Given that no resources associated to the provided deployment exist
    // 
    final ConnectorDeployment deployment = createDeployment(0);
    final List<ManagedConnector> connectors = List.of();
    final List<Secret> secrets = List.of();
    final FleetShardClient fleetShard = ConnectorTestSupport.fleetShard(CLUSTER_ID, connectors, secrets);
    final FleetManagerClient fleetManager = ConnectorTestSupport.fleetManagerClient();
    final FleetShardSyncConfig config = ConnectorTestSupport.config();
    final MeterRegistry registry = Mockito.mock(MeterRegistry.class);
    final ArgumentCaptor<Secret> sc = ArgumentCaptor.forClass(Secret.class);
    final ArgumentCaptor<ManagedConnector> mcc = ArgumentCaptor.forClass(ManagedConnector.class);
    final ConnectorDeploymentProvisioner provisioner = new ConnectorDeploymentProvisioner(config, fleetShard, fleetManager, registry);
    // 
    // When deployment is applied
    // 
    provisioner.provision(deployment);
    verify(fleetShard).createSecret(sc.capture());
    verify(fleetShard).createConnector(mcc.capture());
    // 
    // Then resources must be created according to the deployment
    // 
    assertThat(sc.getValue()).satisfies(val -> {
        assertThat(val.getMetadata().getName()).isEqualTo(Secrets.generateConnectorSecretId(deployment.getId()));
        assertThat(val.getMetadata().getLabels()).containsEntry(LABEL_CLUSTER_ID, CLUSTER_ID).containsEntry(LABEL_CONNECTOR_ID, deployment.getSpec().getConnectorId()).containsEntry(LABEL_DEPLOYMENT_ID, deployment.getId()).containsEntry(LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + deployment.getMetadata().getResourceVersion()).containsKey(LABEL_UOW);
        assertThat(val.getData()).containsKey(Secrets.SECRET_ENTRY_SERVICE_ACCOUNT).containsKey(Secrets.SECRET_ENTRY_CONNECTOR);
        var serviceAccountNode = Secrets.extract(val, Secrets.SECRET_ENTRY_SERVICE_ACCOUNT, ServiceAccount.class);
        assertThat(serviceAccountNode.getClientSecret()).isEqualTo(deployment.getSpec().getServiceAccount().getClientSecret());
        assertThat(serviceAccountNode.getClientId()).isEqualTo(deployment.getSpec().getServiceAccount().getClientId());
        var connectorNode = Secrets.extract(val, Secrets.SECRET_ENTRY_CONNECTOR);
        assertThatJson(Secrets.extract(val, Secrets.SECRET_ENTRY_CONNECTOR)).inPath("connector.foo").isEqualTo("connector-foo");
        assertThatJson(connectorNode).inPath("kafka.topic").isEqualTo("kafka-foo");
        var metaNode = Secrets.extract(val, Secrets.SECRET_ENTRY_META);
        assertThatJson(metaNode).isObject().containsKey("connector_type").containsKey("connector_image").containsKey("kamelets").containsKey("operators");
    });
    assertThat(mcc.getValue()).satisfies(val -> {
        assertThat(val.getMetadata().getName()).isEqualTo(Connectors.generateConnectorId(deployment.getId()));
        assertThat(val.getMetadata().getLabels()).containsEntry(LABEL_CLUSTER_ID, CLUSTER_ID).containsEntry(LABEL_CONNECTOR_ID, deployment.getSpec().getConnectorId()).containsEntry(LABEL_DEPLOYMENT_ID, deployment.getId()).containsKey(LABEL_UOW);
        assertThat(val.getSpec().getDeployment()).satisfies(d -> {
            assertThat(d.getSecret()).isEqualTo(sc.getValue().getMetadata().getName());
            assertThat(d.getUnitOfWork()).isNotEmpty().isEqualTo(sc.getValue().getMetadata().getLabels().get(LABEL_UOW));
            assertThat(d.getKafka().getUrl()).isNotEmpty().isEqualTo(deployment.getSpec().getKafka().getUrl());
        });
    });
}

Example 5 with Connectors

use of org.bf2.cos.fleetshard.support.resources.Connectors in project cos-fleetshard by bf2fc6cc711aee1a0c2a.

the class ConnectorTestSupport method config.

public static FleetShardSyncConfig config() {
    FleetShardSyncConfig answer = Mockito.mock(FleetShardSyncConfig.class);
    when(answer.connectors()).thenAnswer(invocation -> {
        var connectors = Mockito.mock(FleetShardSyncConfig.Connectors.class);
        when(connectors.annotations()).thenReturn(Collections.emptyMap());
        when(connectors.labels()).thenReturn(Collections.emptyMap());
        return connectors;
    });
    when(answer.imagePullSecretsName()).thenAnswer(invocation -> {
        return "foo";
    });
    when(answer.namespace()).thenAnswer(invocation -> {
        return "bar";
    });
    when(answer.metrics()).thenAnswer(invocation -> {
        var metrics = Mockito.mock(FleetShardSyncConfig.Metrics.class);
        when(metrics.baseName()).thenReturn("base");
        return metrics;
    });
    when(answer.quota()).thenAnswer(invocation -> {
        return Mockito.mock(FleetShardSyncConfig.Quota.class);
    });
    when(answer.quota().defaultLimits()).thenAnswer(invocation -> {
        return Mockito.mock(FleetShardSyncConfig.DefaultLimits.class);
    });
    when(answer.quota().defaultRequest()).thenAnswer(invocation -> {
        return Mockito.mock(FleetShardSyncConfig.DefaultRequest.class);
    });
    return answer;
}