Examples with KafkaConnector io.strimzi.api.kafka.model.KafkaConnector used on opensource projects

Example 66 with KafkaConnector

use of io.strimzi.api.kafka.model.KafkaConnector in project strimzi by strimzi.

the class ConnectorMockTest method setup.

@SuppressWarnings({ "checkstyle:MethodLength" })
@BeforeEach
public void setup(VertxTestContext testContext) {
    vertx = Vertx.vertx();
    client = new MockKube().withCustomResourceDefinition(Crds.kafkaConnect(), KafkaConnect.class, KafkaConnectList.class, KafkaConnect::getStatus, KafkaConnect::setStatus).end().withCustomResourceDefinition(Crds.kafkaConnector(), KafkaConnector.class, KafkaConnectorList.class, KafkaConnector::getStatus, KafkaConnector::setStatus).end().build();
    PlatformFeaturesAvailability pfa = new PlatformFeaturesAvailability(true, KubernetesVersion.V1_18);
    setupMockConnectAPI();
    ResourceOperatorSupplier ros = new ResourceOperatorSupplier(vertx, client, new ZookeeperLeaderFinder(vertx, // Retry up to 3 times (4 attempts), with overall max delay of 35000ms
    () -> new BackOff(5_000, 2, 4)), new DefaultAdminClientProvider(), new DefaultZookeeperScalerProvider(), ResourceUtils.metricsProvider(), pfa, FeatureGates.NONE, 10_000);
    ClusterOperatorConfig config = ClusterOperatorConfig.fromMap(map(ClusterOperatorConfig.STRIMZI_KAFKA_IMAGES, KafkaVersionTestUtils.getKafkaImagesEnvVarString(), ClusterOperatorConfig.STRIMZI_KAFKA_CONNECT_IMAGES, KafkaVersionTestUtils.getKafkaConnectImagesEnvVarString(), ClusterOperatorConfig.STRIMZI_KAFKA_MIRROR_MAKER_2_IMAGES, KafkaVersionTestUtils.getKafkaMirrorMaker2ImagesEnvVarString(), ClusterOperatorConfig.STRIMZI_FULL_RECONCILIATION_INTERVAL_MS, Long.toString(Long.MAX_VALUE)), KafkaVersionTestUtils.getKafkaVersionLookup());
    kafkaConnectOperator = new KafkaConnectAssemblyOperator(vertx, pfa, ros, config, x -> api);
    Checkpoint async = testContext.checkpoint();
    // Fail test if watcher closes for any reason
    kafkaConnectOperator.createWatch(NAMESPACE, e -> testContext.failNow(e)).onComplete(testContext.succeeding()).compose(watch -> AbstractConnectOperator.createConnectorWatch(kafkaConnectOperator, NAMESPACE, null)).onComplete(testContext.succeeding(v -> async.flag()));
}

Example 67 with KafkaConnector

use of io.strimzi.api.kafka.model.KafkaConnector in project strimzi by strimzi.

the class AbstractConnectOperator method createConnectorWatch.

/**
 * Create a watch on {@code KafkaConnector} in the given {@code namespace}.
 * The watcher will:
 * <ul>
 * <li>{@linkplain #reconcileConnectors(Reconciliation, CustomResource, KafkaConnectStatus, boolean, String, OrderedProperties)} on the KafkaConnect
 * identified by {@code KafkaConnector.metadata.labels[strimzi.io/cluster]}.</li>
 * <li>The {@code KafkaConnector} status is updated with the result.</li>
 * </ul>
 * @param connectOperator The operator for {@code KafkaConnect}.
 * @param watchNamespaceOrWildcard The namespace to watch.
 * @param selectorLabels Selector labels for filtering the custom resources
 *
 * @return A future which completes when the watch has been set up.
 */
public static Future<Void> createConnectorWatch(AbstractConnectOperator<KubernetesClient, KafkaConnect, KafkaConnectList, Resource<KafkaConnect>, KafkaConnectSpec, KafkaConnectStatus> connectOperator, String watchNamespaceOrWildcard, Labels selectorLabels) {
    Optional<LabelSelector> selector = (selectorLabels == null || selectorLabels.toMap().isEmpty()) ? Optional.empty() : Optional.of(new LabelSelector(null, selectorLabels.toMap()));
    return Util.async(connectOperator.vertx, () -> {
        connectOperator.connectorOperator.watch(watchNamespaceOrWildcard, new Watcher<KafkaConnector>() {

            @Override
            public void eventReceived(Action action, KafkaConnector kafkaConnector) {
                String connectorName = kafkaConnector.getMetadata().getName();
                String connectorNamespace = kafkaConnector.getMetadata().getNamespace();
                String connectorKind = kafkaConnector.getKind();
                String connectName = kafkaConnector.getMetadata().getLabels() == null ? null : kafkaConnector.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL);
                String connectNamespace = connectorNamespace;
                switch(action) {
                    case ADDED:
                    case DELETED:
                    case MODIFIED:
                        if (connectName != null) {
                            // Check whether a KafkaConnect exists
                            connectOperator.resourceOperator.getAsync(connectNamespace, connectName).compose(connect -> {
                                KafkaConnectApi apiClient = connectOperator.connectClientProvider.apply(connectOperator.vertx);
                                if (connect == null) {
                                    Reconciliation r = new Reconciliation("connector-watch", connectOperator.kind(), kafkaConnector.getMetadata().getNamespace(), connectName);
                                    updateStatus(r, noConnectCluster(connectNamespace, connectName), kafkaConnector, connectOperator.connectorOperator);
                                    LOGGER.infoCr(r, "{} {} in namespace {} was {}, but Connect cluster {} does not exist", connectorKind, connectorName, connectorNamespace, action, connectName);
                                    return Future.succeededFuture();
                                } else {
                                    // grab the lock and call reconcileConnectors()
                                    // (i.e. short circuit doing a whole KafkaConnect reconciliation).
                                    Reconciliation reconciliation = new Reconciliation("connector-watch", connectOperator.kind(), kafkaConnector.getMetadata().getNamespace(), connectName);
                                    if (!Util.matchesSelector(selector, connect)) {
                                        LOGGER.debugCr(reconciliation, "{} {} in namespace {} was {}, but Connect cluster {} does not match label selector {} and will be ignored", connectorKind, connectorName, connectorNamespace, action, connectName, selectorLabels);
                                        return Future.succeededFuture();
                                    } else if (connect.getSpec() != null && connect.getSpec().getReplicas() == 0) {
                                        LOGGER.infoCr(reconciliation, "{} {} in namespace {} was {}, but Connect cluster {} has 0 replicas", connectorKind, connectorName, connectorNamespace, action, connectName);
                                        updateStatus(reconciliation, zeroReplicas(connectNamespace, connectName), kafkaConnector, connectOperator.connectorOperator);
                                        return Future.succeededFuture();
                                    } else {
                                        LOGGER.infoCr(reconciliation, "{} {} in namespace {} was {}", connectorKind, connectorName, connectorNamespace, action);
                                        return connectOperator.withLock(reconciliation, LOCK_TIMEOUT_MS, () -> connectOperator.reconcileConnectorAndHandleResult(reconciliation, KafkaConnectResources.qualifiedServiceName(connectName, connectNamespace), apiClient, isUseResources(connect), kafkaConnector.getMetadata().getName(), action == Action.DELETED ? null : kafkaConnector).compose(reconcileResult -> {
                                            LOGGER.infoCr(reconciliation, "reconciled");
                                            return Future.succeededFuture(reconcileResult);
                                        }));
                                    }
                                }
                            });
                        } else {
                            updateStatus(new Reconciliation("connector-watch", connectOperator.kind(), kafkaConnector.getMetadata().getNamespace(), null), new InvalidResourceException("Resource lacks label '" + Labels.STRIMZI_CLUSTER_LABEL + "': No connect cluster in which to create this connector."), kafkaConnector, connectOperator.connectorOperator);
                        }
                        break;
                    case ERROR:
                        LOGGER.errorCr(new Reconciliation("connector-watch", connectorKind, connectName, connectorNamespace), "Failed {} {} in namespace {} ", connectorKind, connectorName, connectorNamespace);
                        break;
                    default:
                        LOGGER.errorCr(new Reconciliation("connector-watch", connectorKind, connectName, connectorNamespace), "Unknown action: {} {} in namespace {}", connectorKind, connectorName, connectorNamespace);
                }
            }

            @Override
            public void onClose(WatcherException e) {
                if (e != null) {
                    throw new KubernetesClientException(e.getMessage());
                }
            }
        });
        return null;
    });
}

Example 68 with KafkaConnector

use of io.strimzi.api.kafka.model.KafkaConnector in project strimzi by strimzi.

the class AbstractConnectOperator method removeAnnotation.

/**
 * Patches the KafkaConnector CR to remove the supplied annotation.
 */
private Future<Void> removeAnnotation(Reconciliation reconciliation, KafkaConnector resource, String annotationKey) {
    LOGGER.debugCr(reconciliation, "Removing annotation {}", annotationKey);
    KafkaConnector patchedKafkaConnector = new KafkaConnectorBuilder(resource).editMetadata().removeFromAnnotations(annotationKey).endMetadata().build();
    return connectorOperator.patchAsync(reconciliation, patchedKafkaConnector).compose(ignored -> Future.succeededFuture());
}

Example 69 with KafkaConnector

use of io.strimzi.api.kafka.model.KafkaConnector in project strimzi by strimzi.

the class AbstractConnectOperator method updateStatus.

public static void updateStatus(Reconciliation reconciliation, Throwable error, KafkaConnector kafkaConnector2, CrdOperator<?, KafkaConnector, ?> connectorOperations) {
    KafkaConnectorStatus status = new KafkaConnectorStatus();
    StatusUtils.setStatusConditionAndObservedGeneration(kafkaConnector2, status, error);
    StatusDiff diff = new StatusDiff(kafkaConnector2.getStatus(), status);
    if (!diff.isEmpty()) {
        KafkaConnector copy = new KafkaConnectorBuilder(kafkaConnector2).build();
        copy.setStatus(status);
        connectorOperations.updateStatusAsync(reconciliation, copy);
    }
}

Example 70 with KafkaConnector

use of io.strimzi.api.kafka.model.KafkaConnector in project strimzi by strimzi.

the class AbstractConnectOperator method maybeUpdateStatusCommon.

/**
 * Updates the Status field of the KafkaConnect or KafkaConnector CR. It diffs the desired status against the current status and calls
 * the update only when there is any difference in non-timestamp fields.
 *
 * @param resource The CR of KafkaConnect or KafkaConnector
 * @param reconciliation Reconciliation information
 * @param desiredStatus The KafkaConnectStatus or KafkaConnectorStatus which should be set
 *
 * @return
 */
protected <T extends CustomResource<?, S>, S extends Status, L extends CustomResourceList<T>> Future<Void> maybeUpdateStatusCommon(CrdOperator<KubernetesClient, T, L> resourceOperator, T resource, Reconciliation reconciliation, S desiredStatus, BiFunction<T, S, T> copyWithStatus) {
    Promise<Void> updateStatusPromise = Promise.promise();
    resourceOperator.getAsync(resource.getMetadata().getNamespace(), resource.getMetadata().getName()).onComplete(getRes -> {
        if (getRes.succeeded()) {
            T fetchedResource = getRes.result();
            if (fetchedResource != null) {
                if ((!(fetchedResource instanceof KafkaConnector)) && (!(fetchedResource instanceof KafkaMirrorMaker2))) {
                    LOGGER.warnCr(reconciliation, "{} {} needs to be upgraded from version {} to 'v1beta1' to use the status field", fetchedResource.getKind(), fetchedResource.getMetadata().getName(), fetchedResource.getApiVersion());
                    updateStatusPromise.complete();
                } else {
                    S currentStatus = fetchedResource.getStatus();
                    StatusDiff ksDiff = new StatusDiff(currentStatus, desiredStatus);
                    if (!ksDiff.isEmpty()) {
                        T resourceWithNewStatus = copyWithStatus.apply(fetchedResource, desiredStatus);
                        resourceOperator.updateStatusAsync(reconciliation, resourceWithNewStatus).onComplete(updateRes -> {
                            if (updateRes.succeeded()) {
                                LOGGER.debugCr(reconciliation, "Completed status update");
                                updateStatusPromise.complete();
                            } else {
                                LOGGER.errorCr(reconciliation, "Failed to update status", updateRes.cause());
                                updateStatusPromise.fail(updateRes.cause());
                            }
                        });
                    } else {
                        LOGGER.debugCr(reconciliation, "Status did not change");
                        updateStatusPromise.complete();
                    }
                }
            } else {
                LOGGER.errorCr(reconciliation, "Current {} resource not found", resource.getKind());
                updateStatusPromise.fail("Current " + resource.getKind() + " resource not found");
            }
        } else {
            LOGGER.errorCr(reconciliation, "Failed to get the current {} resource and its status", resource.getKind(), getRes.cause());
            updateStatusPromise.fail(getRes.cause());
        }
    });
    return updateStatusPromise.future();
}