Examples with KafkaMirrorMaker2ClusterSpec io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec used on opensource projects

Example 46 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2Cluster method fromCrd.

/**
 * Creates instance of KafkaMirrorMaker2Cluster from CRD definition.
 *
 * @param reconciliation    The reconciliation
 * @param kafkaMirrorMaker2 The Custom Resource based on which the cluster model should be created.
 * @param versions The image versions for MirrorMaker 2.0 clusters.
 * @return The MirrorMaker 2.0 cluster model.
 */
public static KafkaMirrorMaker2Cluster fromCrd(Reconciliation reconciliation, KafkaMirrorMaker2 kafkaMirrorMaker2, KafkaVersion.Lookup versions) {
    KafkaMirrorMaker2Cluster cluster = new KafkaMirrorMaker2Cluster(reconciliation, kafkaMirrorMaker2);
    KafkaMirrorMaker2Spec spec = kafkaMirrorMaker2.getSpec();
    cluster.setOwnerReference(kafkaMirrorMaker2);
    cluster.setImage(versions.kafkaMirrorMaker2Version(spec.getImage(), spec.getVersion()));
    List<KafkaMirrorMaker2ClusterSpec> clustersList = ModelUtils.asListOrEmptyList(spec.getClusters());
    cluster.setClusters(clustersList);
    KafkaMirrorMaker2ClusterSpec connectCluster = new KafkaMirrorMaker2ClusterSpecBuilder().build();
    String connectClusterAlias = spec.getConnectCluster();
    if (connectClusterAlias != null) {
        connectCluster = clustersList.stream().filter(clustersListItem -> spec.getConnectCluster().equals(clustersListItem.getAlias())).findFirst().orElseThrow(() -> new InvalidResourceException("connectCluster with alias " + connectClusterAlias + " cannot be found in the list of clusters at spec.clusters"));
    }
    cluster.setConfiguration(new KafkaMirrorMaker2Configuration(reconciliation, connectCluster.getConfig().entrySet()));
    KafkaMirrorMaker2Cluster mm2 = fromSpec(reconciliation, buildKafkaConnectSpec(spec, connectCluster), versions, cluster);
    mm2.templatePodLabels = Util.mergeLabelsOrAnnotations(mm2.templatePodLabels, DEFAULT_POD_LABELS);
    return mm2;
}

Example 47 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2ClusterTest method testGenerateDeploymentWithTlsSameSecret.

@ParallelTest
public void testGenerateDeploymentWithTlsSameSecret() {
    KafkaMirrorMaker2ClusterSpec targetClusterWithTlsAuth = new KafkaMirrorMaker2ClusterSpecBuilder(this.targetCluster).editOrNewTls().addToTrustedCertificates(new CertSecretSourceBuilder().withSecretName("my-secret").withCertificate("cert.crt").build()).endTls().withAuthentication(new KafkaClientAuthenticationTlsBuilder().withNewCertificateAndKey().withSecretName("my-secret").withCertificate("user.crt").withKey("user.key").endCertificateAndKey().build()).build();
    KafkaMirrorMaker2 resource = new KafkaMirrorMaker2Builder(this.resource).editSpec().withClusters(targetClusterWithTlsAuth).endSpec().build();
    KafkaMirrorMaker2Cluster kmm2 = KafkaMirrorMaker2Cluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, resource, VERSIONS);
    Deployment dep = kmm2.generateDeployment(emptyMap(), true, null, null);
    // 3 = 1 volume from logging/metrics + 2 from above cert mounted for connect and for connectors
    assertThat(dep.getSpec().getTemplate().getSpec().getVolumes().size(), is(4));
    assertThat(dep.getSpec().getTemplate().getSpec().getVolumes().get(2).getName(), is("my-secret"));
}

Example 48 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2ClusterTest method testGenerateDeploymentWithScramSha512Auth.

@ParallelTest
public void testGenerateDeploymentWithScramSha512Auth() {
    KafkaMirrorMaker2ClusterSpec targetClusterWithScramSha512Auth = new KafkaMirrorMaker2ClusterSpecBuilder(this.targetCluster).withNewKafkaClientAuthenticationScramSha512().withUsername("user1").withNewPasswordSecret().withSecretName("user1-secret").withPassword("password").endPasswordSecret().endKafkaClientAuthenticationScramSha512().build();
    KafkaMirrorMaker2 resource = new KafkaMirrorMaker2Builder(this.resource).editSpec().withClusters(targetClusterWithScramSha512Auth).endSpec().build();
    KafkaMirrorMaker2Cluster kmm2 = KafkaMirrorMaker2Cluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, resource, VERSIONS);
    Deployment dep = kmm2.generateDeployment(emptyMap(), true, null, null);
    assertThat(dep.getSpec().getTemplate().getSpec().getVolumes().get(2).getName(), is("user1-secret"));
    Container cont = getContainer(dep);
    assertThat(cont.getVolumeMounts().get(2).getMountPath(), is(KafkaMirrorMaker2Cluster.PASSWORD_VOLUME_MOUNT + "user1-secret"));
    assertThat(AbstractModel.containerEnvVars(cont).get(KafkaMirrorMaker2Cluster.ENV_VAR_KAFKA_CONNECT_SASL_PASSWORD_FILE), is("user1-secret/password"));
    assertThat(AbstractModel.containerEnvVars(cont).get(KafkaMirrorMaker2Cluster.ENV_VAR_KAFKA_CONNECT_SASL_USERNAME), is("user1"));
    assertThat(AbstractModel.containerEnvVars(cont).get(KafkaMirrorMaker2Cluster.ENV_VAR_KAFKA_CONNECT_SASL_MECHANISM), is("scram-sha-512"));
}

Example 49 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class KafkaMirrorMaker2AssemblyOperatorTest method testTopicsGroupsBlacklist.

@Test
@SuppressWarnings("deprecation")
public void testTopicsGroupsBlacklist(VertxTestContext context) {
    String kmm2Name = "foo";
    String sourceNamespace = "source-ns";
    String targetNamespace = "target-ns";
    String sourceClusterAlias = "my-cluster-src";
    String targetClusterAlias = "my-cluster-tgt";
    String excludedTopicList = "excludedTopic0,excludedTopic1";
    String excludedGroupList = "excludedGroup0,excludedGroup1";
    ResourceOperatorSupplier supplier = ResourceUtils.supplierWithMocks(true);
    KafkaMirrorMaker2 kmm2 = ResourceUtils.createEmptyKafkaMirrorMaker2(targetNamespace, kmm2Name);
    ArgumentCaptor<KafkaMirrorMaker2> mirrorMaker2Captor = createMirrorMaker2CaptorMock(targetNamespace, kmm2Name, kmm2, supplier);
    KafkaConnectApi mockConnectClient = createConnectClientMock();
    KafkaMirrorMaker2ClusterSpec sourceCluster = new KafkaMirrorMaker2ClusterSpecBuilder(true).withAlias(sourceClusterAlias).withBootstrapServers(sourceClusterAlias + "." + sourceNamespace + ".svc:9092").build();
    KafkaMirrorMaker2ClusterSpec targetCluster = new KafkaMirrorMaker2ClusterSpecBuilder(true).withAlias(targetClusterAlias).withBootstrapServers(targetClusterAlias + "." + targetNamespace + ".svc:9092").build();
    kmm2.getSpec().setClusters(List.of(sourceCluster, targetCluster));
    KafkaMirrorMaker2MirrorSpec deprecatedMirrorConnector = new KafkaMirrorMaker2MirrorSpecBuilder().withSourceCluster(sourceClusterAlias).withTargetCluster(targetClusterAlias).withTopicsBlacklistPattern(excludedTopicList).withGroupsBlacklistPattern(excludedGroupList).build();
    kmm2.getSpec().setMirrors(List.of(deprecatedMirrorConnector));
    KafkaMirrorMaker2AssemblyOperator mm2AssemblyOperator = new KafkaMirrorMaker2AssemblyOperator(vertx, new PlatformFeaturesAvailability(true, kubernetesVersion), supplier, ResourceUtils.dummyClusterOperatorConfig(VERSIONS), x -> mockConnectClient);
    Checkpoint async = context.checkpoint();
    KafkaMirrorMaker2Cluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kmm2, VERSIONS);
    mm2AssemblyOperator.reconcile(new Reconciliation("test-blacklist", KafkaMirrorMaker2.RESOURCE_KIND, targetNamespace, kmm2Name)).onComplete(context.succeeding(v -> context.verify(() -> {
        KafkaMirrorMaker2MirrorSpec capturedMirrorConnector = mirrorMaker2Captor.getAllValues().get(0).getSpec().getMirrors().get(0);
        assertThat(capturedMirrorConnector.getTopicsBlacklistPattern(), is(excludedTopicList));
        assertThat(capturedMirrorConnector.getGroupsBlacklistPattern(), is(excludedGroupList));
        async.flag();
    })));
}

Example 50 with KafkaMirrorMaker2ClusterSpec

use of io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec in project strimzi by strimzi.

the class OauthPlainIsolatedST method testProducerConsumerMirrorMaker2.

@IsolatedTest("Using more tha one Kafka cluster in one namespace")
@Tag(MIRROR_MAKER2)
@Tag(CONNECT_COMPONENTS)
@Tag(NODEPORT_SUPPORTED)
void testProducerConsumerMirrorMaker2(ExtensionContext extensionContext) {
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String producerName = OAUTH_PRODUCER_NAME + "-" + clusterName;
    String consumerName = OAUTH_CONSUMER_NAME + "-" + clusterName;
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    KafkaOauthClients oauthExampleClients = new KafkaOauthClientsBuilder().withNamespaceName(INFRA_NAMESPACE).withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(oauthClusterName)).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).withOauthClientId(OAUTH_CLIENT_NAME).withOauthClientSecret(OAUTH_CLIENT_SECRET).withOauthTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).build();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(oauthClusterName, topicName, INFRA_NAMESPACE).build());
    resourceManager.createResource(extensionContext, oauthExampleClients.producerStrimziOauthPlain());
    ClientUtils.waitForClientSuccess(producerName, INFRA_NAMESPACE, MESSAGE_COUNT);
    JobUtils.deleteJobWithWait(INFRA_NAMESPACE, producerName);
    resourceManager.createResource(extensionContext, oauthExampleClients.consumerStrimziOauthPlain());
    ClientUtils.waitForClientSuccess(consumerName, INFRA_NAMESPACE, MESSAGE_COUNT);
    JobUtils.deleteJobWithWait(INFRA_NAMESPACE, consumerName);
    String kafkaSourceClusterName = oauthClusterName;
    String kafkaTargetClusterName = clusterName + "-target";
    // mirror maker 2 adding prefix to mirrored topic for in this case mirrotopic will be : my-cluster.my-topic
    String kafkaTargetClusterTopicName = kafkaSourceClusterName + "." + topicName;
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaTargetClusterName, 1, 1).editMetadata().withNamespace(INFRA_NAMESPACE).endMetadata().editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.PLAIN_LISTENER_DEFAULT_NAME).withPort(9092).withType(KafkaListenerType.INTERNAL).withTls(false).withNewKafkaListenerAuthenticationOAuth().withValidIssuerUri(keycloakInstance.getValidIssuerUri()).withJwksEndpointUri(keycloakInstance.getJwksEndpointUri()).withJwksExpirySeconds(keycloakInstance.getJwksExpireSeconds()).withJwksRefreshSeconds(keycloakInstance.getJwksRefreshSeconds()).withUserNameClaim(keycloakInstance.getUserNameClaim()).endKafkaListenerAuthenticationOAuth().build(), new GenericKafkaListenerBuilder().withName(Constants.EXTERNAL_LISTENER_DEFAULT_NAME).withPort(9094).withType(KafkaListenerType.NODEPORT).withTls(false).withNewKafkaListenerAuthenticationOAuth().withValidIssuerUri(keycloakInstance.getValidIssuerUri()).withJwksExpirySeconds(keycloakInstance.getJwksExpireSeconds()).withJwksRefreshSeconds(keycloakInstance.getJwksRefreshSeconds()).withJwksEndpointUri(keycloakInstance.getJwksEndpointUri()).withUserNameClaim(keycloakInstance.getUserNameClaim()).endKafkaListenerAuthenticationOAuth().build()).endKafka().endSpec().build());
    // Deploy Mirror Maker 2.0 with oauth
    KafkaMirrorMaker2ClusterSpec sourceClusterWithOauth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaSourceClusterName).withConfig(connectorConfig).withBootstrapServers(KafkaResources.plainBootstrapAddress(kafkaSourceClusterName)).withNewKafkaClientAuthenticationOAuth().withTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).withClientId("kafka-mirror-maker-2").withNewClientSecret().withSecretName(MIRROR_MAKER_2_OAUTH_SECRET).withKey(OAUTH_KEY).endClientSecret().withConnectTimeoutSeconds(CONNECT_TIMEOUT_S).withReadTimeoutSeconds(READ_TIMEOUT_S).endKafkaClientAuthenticationOAuth().build();
    KafkaMirrorMaker2ClusterSpec targetClusterWithOauth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaTargetClusterName).withConfig(connectorConfig).withBootstrapServers(KafkaResources.plainBootstrapAddress(kafkaTargetClusterName)).withNewKafkaClientAuthenticationOAuth().withTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).withClientId("kafka-mirror-maker-2").withNewClientSecret().withSecretName(MIRROR_MAKER_2_OAUTH_SECRET).withKey(OAUTH_KEY).endClientSecret().withConnectTimeoutSeconds(CONNECT_TIMEOUT_S).withReadTimeoutSeconds(READ_TIMEOUT_S).endKafkaClientAuthenticationOAuth().build();
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(oauthClusterName, kafkaTargetClusterName, kafkaSourceClusterName, 1, false).editMetadata().withNamespace(INFRA_NAMESPACE).endMetadata().editSpec().withClusters(sourceClusterWithOauth, targetClusterWithOauth).editFirstMirror().withSourceCluster(kafkaSourceClusterName).endMirror().endSpec().build());
    final String kafkaMirrorMaker2PodName = kubeClient(INFRA_NAMESPACE).listPods(INFRA_NAMESPACE, oauthClusterName, Labels.STRIMZI_KIND_LABEL, KafkaMirrorMaker2.RESOURCE_KIND).get(0).getMetadata().getName();
    final String kafkaMirrorMaker2Logs = KubeClusterResource.cmdKubeClient(INFRA_NAMESPACE).execInCurrentNamespace(Level.DEBUG, "logs", kafkaMirrorMaker2PodName).out();
    verifyOauthConfiguration(kafkaMirrorMaker2Logs);
    TestUtils.waitFor("Waiting for Mirror Maker 2 will copy messages from " + kafkaSourceClusterName + " to " + kafkaTargetClusterName, Duration.ofSeconds(30).toMillis(), Constants.TIMEOUT_FOR_MIRROR_MAKER_COPY_MESSAGES_BETWEEN_BROKERS, () -> {
        LOGGER.info("Deleting the Job {}", consumerName);
        JobUtils.deleteJobWithWait(INFRA_NAMESPACE, consumerName);
        LOGGER.info("Creating new client with new consumer-group and also to point on {} cluster", kafkaTargetClusterName);
        KafkaOauthClients kafkaOauthClientJob = new KafkaOauthClientsBuilder().withNamespaceName(INFRA_NAMESPACE).withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(kafkaTargetClusterName)).withTopicName(kafkaTargetClusterTopicName).withMessageCount(MESSAGE_COUNT).withOauthClientId(OAUTH_CLIENT_NAME).withOauthClientSecret(OAUTH_CLIENT_SECRET).withOauthTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).build();
        resourceManager.createResource(extensionContext, kafkaOauthClientJob.consumerStrimziOauthPlain());
        try {
            ClientUtils.waitForClientSuccess(consumerName, INFRA_NAMESPACE, MESSAGE_COUNT);
            return true;
        } catch (WaitException e) {
            e.printStackTrace();
            return false;
        }
    });
}