use of org.bf2.cos.fleetshard.api.Version in project cos-fleetshard by bf2fc6cc711aee1a0c2a.
the class ConnectorDeploymentProvisioner method createManagedConnector.
private ManagedConnector createManagedConnector(String uow, ConnectorDeployment deployment, HasMetadata owner) {
ManagedConnector connector = fleetShard.getConnector(deployment).orElseGet(() -> {
LOGGER.info("Connector not found (cluster_id: {}, namespace_id: {}, connector_id: {}, deployment_id: {}, resource_version: {}), creating a new one", fleetShard.getClusterId(), deployment.getSpec().getNamespaceId(), deployment.getSpec().getConnectorId(), deployment.getId(), deployment.getMetadata().getResourceVersion());
ManagedConnector answer = new ManagedConnector();
answer.setMetadata(new ObjectMeta());
answer.getMetadata().setNamespace(fleetShard.generateNamespaceId(deployment.getSpec().getNamespaceId()));
answer.getMetadata().setName(Connectors.generateConnectorId(deployment.getId()));
Resources.setLabels(answer, LABEL_CLUSTER_ID, fleetShard.getClusterId(), LABEL_CONNECTOR_ID, deployment.getSpec().getConnectorId(), LABEL_DEPLOYMENT_ID, deployment.getId());
answer.getSpec().setClusterId(fleetShard.getClusterId());
answer.getSpec().setConnectorId(deployment.getSpec().getConnectorId());
answer.getSpec().setDeploymentId(deployment.getId());
return answer;
});
// TODO: change APIs to include a single operator
// move operator one level up
// include full operator info in ConnectorDeployment APIs
ArrayNode operatorsMeta = deployment.getSpec().getShardMetadata().withArray("operators");
if (operatorsMeta.size() != 1) {
throw new IllegalArgumentException("Multiple selectors are not yet supported");
}
OperatorSelector operatorSelector = new OperatorSelector(deployment.getSpec().getOperatorId(), operatorsMeta.get(0).requiredAt("/type").asText(), operatorsMeta.get(0).requiredAt("/version").asText());
if (operatorSelector.getId() == null) {
final OperatorSelector currentSelector = connector.getSpec().getOperatorSelector();
// don't select a new operator if previously set.
if (currentSelector != null && currentSelector.getId() != null) {
operatorSelector.setId(currentSelector.getId());
} else {
Collection<Operator> operators = fleetShard.getOperators().stream().map(mco -> new Operator(mco.getMetadata().getName(), mco.getSpec().getType(), mco.getSpec().getVersion())).collect(Collectors.toList());
OperatorSelectorUtil.assign(operatorSelector, operators).map(Operator::getId).ifPresent(operatorSelector::setId);
}
}
if (operatorSelector.getId() != null) {
Resources.setLabel(connector, LABEL_OPERATOR_ASSIGNED, operatorSelector.getId());
}
if (operatorSelector.getType() != null) {
Resources.setLabel(connector, LABEL_OPERATOR_TYPE, operatorSelector.getType());
}
if (config != null) {
config.connectors().labels().forEach((k, v) -> {
Resources.setLabel(connector, k, v);
});
config.connectors().annotations().forEach((k, v) -> {
Resources.setAnnotation(connector, k, v);
});
}
Resources.setOwnerReferences(connector, owner);
// add resource version to label
Resources.setLabel(connector, LABEL_DEPLOYMENT_RESOURCE_VERSION, "" + deployment.getMetadata().getResourceVersion());
// add uow
Resources.setLabel(connector, LABEL_UOW, uow);
connector.getSpec().getDeployment().setDeploymentResourceVersion(deployment.getMetadata().getResourceVersion());
connector.getSpec().getDeployment().setDesiredState(deployment.getSpec().getDesiredState().getValue());
connector.getSpec().getDeployment().setConnectorTypeId(deployment.getSpec().getConnectorTypeId());
connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
KafkaConnectionSettings kafkaConnectionSettings = deployment.getSpec().getKafka();
if (kafkaConnectionSettings != null) {
connector.getSpec().getDeployment().setKafka(new KafkaSpec(kafkaConnectionSettings.getId(), kafkaConnectionSettings.getUrl()));
}
SchemaRegistryConnectionSettings schemaRegistryConnectionSettings = deployment.getSpec().getSchemaRegistry();
if (schemaRegistryConnectionSettings != null) {
connector.getSpec().getDeployment().setSchemaRegistry(new SchemaRegistrySpec(schemaRegistryConnectionSettings.getId(), schemaRegistryConnectionSettings.getUrl()));
}
connector.getSpec().getDeployment().setConnectorResourceVersion(deployment.getSpec().getConnectorResourceVersion());
connector.getSpec().getDeployment().setSecret(Secrets.generateConnectorSecretId(deployment.getId()));
connector.getSpec().getDeployment().setUnitOfWork(uow);
connector.getSpec().setOperatorSelector(operatorSelector);
LOGGER.info("Provisioning connector namespace: {}, name: {}, revision: {}", connector.getMetadata().getNamespace(), connector.getMetadata().getName(), connector.getSpec().getDeployment().getDeploymentResourceVersion());
try {
return fleetShard.createConnector(connector);
} catch (Exception e) {
LOGGER.warn("", e);
throw e;
}
}
use of org.bf2.cos.fleetshard.api.Version in project cos-fleetshard by bf2fc6cc711aee1a0c2a.
the class NamespaceReaperWithLeftoversTest method namespaceIsProvisioned.
@Test
void namespaceIsProvisioned() {
final String deploymentId = ConfigProvider.getConfig().getValue("test.deployment.id", String.class);
final String statusUrl = "/api/connector_mgmt/v1/agent/kafka_connector_clusters/" + config.cluster().id() + "/status";
final String namespaceName = fleetShardClient.generateNamespaceId(deploymentId);
given().contentType(MediaType.TEXT_PLAIN).body(0L).post("/test/provisioner/namespaces");
until(() -> fleetShardClient.getNamespace(deploymentId), Objects::nonNull);
server.until(putRequestedFor(urlEqualTo(statusUrl)).withHeader(ContentTypeHeader.KEY, equalTo(APPLICATION_JSON)).withRequestBody(jp("$.namespaces.size()", "1")).withRequestBody(jp("$.namespaces[0].phase", Namespaces.PHASE_READY)).withRequestBody(jp("$.namespaces[0].version", "0")).withRequestBody(jp("$.namespaces[0].connectors_deployed", "0")));
final ManagedConnector connector = new ManagedConnectorBuilder().withMetadata(new ObjectMetaBuilder().withName(Connectors.generateConnectorId(deploymentId)).withNamespace(namespaceName).addToLabels(LABEL_CLUSTER_ID, config.cluster().id()).addToLabels(LABEL_CONNECTOR_ID, deploymentId).addToLabels(LABEL_DEPLOYMENT_ID, deploymentId).build()).withSpec(new ManagedConnectorSpecBuilder().withClusterId(config.cluster().id()).withConnectorId(deploymentId).withDeploymentId(deploymentId).withOperatorSelector(new OperatorSelectorBuilder().withId(deploymentId).build()).build()).build();
kubernetesClient.resources(ManagedConnector.class).inNamespace(connector.getMetadata().getNamespace()).create(connector);
untilAsserted(() -> {
assertThat(fleetShardClient.getAllConnectors()).isNotEmpty();
assertThat(fleetShardClient.getConnectors(connector.getMetadata().getNamespace())).isNotEmpty();
});
given().contentType(MediaType.TEXT_PLAIN).body(1L).post("/test/provisioner/namespaces");
server.until(putRequestedFor(urlEqualTo(statusUrl)).withHeader(ContentTypeHeader.KEY, equalTo(APPLICATION_JSON)).withRequestBody(matchingJsonPath("$.namespaces", WireMock.absent())));
}
use of org.bf2.cos.fleetshard.api.Version in project cos-fleetshard by bf2fc6cc711aee1a0c2a.
the class OperatorSelectorUtil method available.
public static Optional<Operator> available(OperatorSelector selector, Collection<Operator> operators) {
if (operators == null) {
return Optional.empty();
}
if (operators.isEmpty()) {
return Optional.empty();
}
final VersionRange range = new VersionRange(selector.getVersion());
final Comparator<Operator> cmp = Comparator.comparing(o -> new Version(o.getVersion()));
return operators.stream().filter(o -> Objects.equals(o.getType(), selector.getType())).filter(o -> versionInRange(range, o)).max(cmp);
}
use of org.bf2.cos.fleetshard.api.Version in project kas-fleetshard by bf2fc6cc711aee1a0c2a.
the class Canary method buildEnvVar.
private List<EnvVar> buildEnvVar(ManagedKafka managedKafka, Deployment current) {
List<EnvVar> envVars = new ArrayList<>(10);
String bootstrap = getBootstrapURL(managedKafka);
envVars.add(new EnvVarBuilder().withName("KAFKA_BOOTSTRAP_SERVERS").withValue(bootstrap).build());
envVars.add(new EnvVarBuilder().withName("RECONCILE_INTERVAL_MS").withValue("5000").build());
envVars.add(new EnvVarBuilder().withName("EXPECTED_CLUSTER_SIZE").withValue(String.valueOf(kafkaCluster.getReplicas(managedKafka))).build());
String kafkaVersion = managedKafka.getSpec().getVersions().getKafka();
// takes the current Kafka version if the canary already exists. During Kafka upgrades it doesn't have to change, as any other clients.
if (current != null) {
Optional<EnvVar> kafkaVersionEnvVar = current.getSpec().getTemplate().getSpec().getContainers().stream().filter(container -> "canary".equals(container.getName())).findFirst().get().getEnv().stream().filter(ev -> "KAFKA_VERSION".equals(ev.getName())).findFirst();
if (kafkaVersionEnvVar.isPresent()) {
kafkaVersion = kafkaVersionEnvVar.get().getValue();
}
}
envVars.add(new EnvVarBuilder().withName("KAFKA_VERSION").withValue(kafkaVersion).build());
envVars.add(new EnvVarBuilder().withName("TZ").withValue("UTC").build());
envVars.add(new EnvVarBuilder().withName("TLS_ENABLED").withValue("true").build());
envVars.add(new EnvVarBuilder().withName("TLS_CA_CERT").withValue("/tmp/tls-ca-cert/ca.crt").build());
// Deprecated
EnvVarSource saramaLogEnabled = new EnvVarSourceBuilder().editOrNewConfigMapKeyRef().withName(CANARY_CONFIG_CONFIGMAP_NAME).withKey("sarama.log.enabled").withOptional(Boolean.TRUE).endConfigMapKeyRef().build();
EnvVarSource verbosityLogLevel = new EnvVarSourceBuilder().editOrNewConfigMapKeyRef().withName(CANARY_CONFIG_CONFIGMAP_NAME).withKey("verbosity.log.level").withOptional(Boolean.TRUE).endConfigMapKeyRef().build();
EnvVarSource goDebug = new EnvVarSourceBuilder().editOrNewConfigMapKeyRef().withName(CANARY_CONFIG_CONFIGMAP_NAME).withKey("go.debug").withOptional(Boolean.TRUE).endConfigMapKeyRef().build();
envVars.add(new EnvVarBuilder().withName("SARAMA_LOG_ENABLED").withValueFrom(saramaLogEnabled).build());
envVars.add(new EnvVarBuilder().withName("VERBOSITY_LOG_LEVEL").withValueFrom(verbosityLogLevel).build());
envVars.add(new EnvVarBuilder().withName("GODEBUG").withValueFrom(goDebug).build());
envVars.add(new EnvVarBuilder().withName("TOPIC").withValue(config.getCanary().getTopic()).build());
envVars.add(new EnvVarBuilder().withName("TOPIC_CONFIG").withValue("retention.ms=600000;segment.bytes=16384").build());
envVars.add(new EnvVarBuilder().withName("CLIENT_ID").withValue(config.getCanary().getClientId()).build());
envVars.add(new EnvVarBuilder().withName("CONSUMER_GROUP_ID").withValue(config.getCanary().getConsumerGroupId()).build());
envVars.add(new EnvVarBuilder().withName("PRODUCER_LATENCY_BUCKETS").withValue(producerLatencyBuckets).build());
envVars.add(new EnvVarBuilder().withName("ENDTOEND_LATENCY_BUCKETS").withValue(endToEndLatencyBuckets).build());
envVars.add(new EnvVarBuilder().withName("CONNECTION_CHECK_LATENCY_BUCKETS").withValue(connectionCheckLatencyBuckets).build());
envVars.add(new EnvVarBuilder().withName("DYNAMIC_CONFIG_FILE").withValue(CANARY_DYNAMIC_CONFIG_JSON.toString()).build());
if (SecuritySecretManager.isCanaryServiceAccountPresent(managedKafka)) {
envVars.add(new EnvVarBuilder().withName("SASL_MECHANISM").withValue("PLAIN").build());
addEnvVarFromSecret(envVars, "SASL_USER", SecuritySecretManager.canarySaslSecretName(managedKafka), SecuritySecretManager.SASL_PRINCIPAL);
addEnvVarFromSecret(envVars, "SASL_PASSWORD", SecuritySecretManager.canarySaslSecretName(managedKafka), SecuritySecretManager.SASL_PASSWORD);
}
envVars.add(new EnvVarBuilder().withName("STATUS_TIME_WINDOW_MS").withValue(String.valueOf(statusTimeWindowMs)).build());
return envVars;
}
use of org.bf2.cos.fleetshard.api.Version in project kas-fleetshard by bf2fc6cc711aee1a0c2a.
the class ManagedKafkaController method validity.
/**
* Run a validity check on the ManagedKafka custom resource
*
* @param managedKafka ManagedKafka custom resource to validate
* @return readiness indicating an error in the ManagedKafka custom resource, empty Optional otherwise
*/
private Optional<OperandReadiness> validity(ManagedKafka managedKafka) {
String message = null;
StrimziVersionStatus strimziVersion = this.strimziManager.getStrimziVersion(managedKafka.getSpec().getVersions().getStrimzi());
if (strimziVersion == null) {
message = String.format("The requested Strimzi version %s is not supported", managedKafka.getSpec().getVersions().getStrimzi());
} else {
if (!strimziVersion.getKafkaVersions().contains(managedKafka.getSpec().getVersions().getKafka())) {
message = String.format("The requested Kafka version %s is not supported by the Strimzi version %s", managedKafka.getSpec().getVersions().getKafka(), strimziVersion.getVersion());
} else if (managedKafka.getSpec().getVersions().getKafkaIbp() != null && !strimziVersion.getKafkaIbpVersions().contains(managedKafka.getSpec().getVersions().getKafkaIbp())) {
message = String.format("The requested Kafka inter broker protocol version %s is not supported by the Strimzi version %s", managedKafka.getSpec().getVersions().getKafkaIbp(), strimziVersion.getVersion());
}
}
if (message != null) {
log.error(message);
return Optional.of(new OperandReadiness(Status.False, Reason.Error, message));
}
return Optional.empty();
}
Aggregations