Search in sources :

Example 21 with CertSecretSource

use of io.strimzi.api.kafka.model.CertSecretSource in project strimzi-kafka-operator by strimzi.

the class KafkaBridgeClusterTest method testGenerateDeploymentWithOAuthWithTls.

@ParallelTest
public void testGenerateDeploymentWithOAuthWithTls() {
    CertSecretSource cert1 = new CertSecretSourceBuilder().withSecretName("first-certificate").withCertificate("ca.crt").build();
    CertSecretSource cert2 = new CertSecretSourceBuilder().withSecretName("second-certificate").withCertificate("tls.crt").build();
    CertSecretSource cert3 = new CertSecretSourceBuilder().withSecretName("first-certificate").withCertificate("ca2.crt").build();
    KafkaBridge resource = new KafkaBridgeBuilder(this.resource).editSpec().withAuthentication(new KafkaClientAuthenticationOAuthBuilder().withClientId("my-client-id").withTokenEndpointUri("http://my-oauth-server").withNewClientSecret().withSecretName("my-secret-secret").withKey("my-secret-key").endClientSecret().withDisableTlsHostnameVerification(true).withTlsTrustedCertificates(cert1, cert2, cert3).build()).endSpec().build();
    KafkaBridgeCluster kb = KafkaBridgeCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, resource, VERSIONS);
    Deployment dep = kb.generateDeployment(emptyMap(), true, null, null);
    Container cont = dep.getSpec().getTemplate().getSpec().getContainers().get(0);
    assertThat(cont.getEnv().stream().filter(var -> KafkaBridgeCluster.ENV_VAR_KAFKA_BRIDGE_SASL_MECHANISM.equals(var.getName())).findFirst().orElseThrow().getValue(), is("oauth"));
    assertThat(cont.getEnv().stream().filter(var -> KafkaBridgeCluster.ENV_VAR_KAFKA_BRIDGE_OAUTH_CLIENT_SECRET.equals(var.getName())).findFirst().orElseThrow().getValueFrom().getSecretKeyRef().getName(), is("my-secret-secret"));
    assertThat(cont.getEnv().stream().filter(var -> KafkaBridgeCluster.ENV_VAR_KAFKA_BRIDGE_OAUTH_CLIENT_SECRET.equals(var.getName())).findFirst().orElseThrow().getValueFrom().getSecretKeyRef().getKey(), is("my-secret-key"));
    assertThat(cont.getEnv().stream().filter(var -> KafkaBridgeCluster.ENV_VAR_KAFKA_BRIDGE_OAUTH_CONFIG.equals(var.getName())).findFirst().orElseThrow().getValue().trim(), is(String.format("%s=\"%s\" %s=\"%s\" %s=\"%s\"", ClientConfig.OAUTH_CLIENT_ID, "my-client-id", ClientConfig.OAUTH_TOKEN_ENDPOINT_URI, "http://my-oauth-server", ServerConfig.OAUTH_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM, "")));
    // Volume mounts
    assertThat(cont.getVolumeMounts().stream().filter(mount -> "oauth-certs-0".equals(mount.getName())).findFirst().orElseThrow().getMountPath(), is(KafkaBridgeCluster.OAUTH_TLS_CERTS_BASE_VOLUME_MOUNT + "/first-certificate-0"));
    assertThat(cont.getVolumeMounts().stream().filter(mount -> "oauth-certs-1".equals(mount.getName())).findFirst().orElseThrow().getMountPath(), is(KafkaBridgeCluster.OAUTH_TLS_CERTS_BASE_VOLUME_MOUNT + "/second-certificate-1"));
    assertThat(cont.getVolumeMounts().stream().filter(mount -> "oauth-certs-2".equals(mount.getName())).findFirst().orElseThrow().getMountPath(), is(KafkaBridgeCluster.OAUTH_TLS_CERTS_BASE_VOLUME_MOUNT + "/first-certificate-2"));
    // Volumes
    List<KeyToPath> cert1Items = dep.getSpec().getTemplate().getSpec().getVolumes().stream().filter(vol -> "oauth-certs-0".equals(vol.getName())).findFirst().orElseThrow().getSecret().getItems();
    assertThat(cert1Items.size(), is(1));
    assertThat(cert1Items.get(0).getKey(), is("ca.crt"));
    assertThat(cert1Items.get(0).getPath(), is("tls.crt"));
    List<KeyToPath> cert2Items = dep.getSpec().getTemplate().getSpec().getVolumes().stream().filter(vol -> "oauth-certs-1".equals(vol.getName())).findFirst().orElseThrow().getSecret().getItems();
    assertThat(cert2Items.size(), is(1));
    assertThat(cert2Items.get(0).getKey(), is("tls.crt"));
    assertThat(cert2Items.get(0).getPath(), is("tls.crt"));
    List<KeyToPath> cert3Items = dep.getSpec().getTemplate().getSpec().getVolumes().stream().filter(vol -> "oauth-certs-2".equals(vol.getName())).findFirst().orElseThrow().getSecret().getItems();
    assertThat(cert3Items.size(), is(1));
    assertThat(cert3Items.get(0).getKey(), is("ca2.crt"));
    assertThat(cert3Items.get(0).getPath(), is("tls.crt"));
}
Also used : KafkaClientAuthenticationOAuthBuilder(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuthBuilder) Quantity(io.fabric8.kubernetes.api.model.Quantity) CoreMatchers.is(org.hamcrest.CoreMatchers.is) IntOrString(io.fabric8.kubernetes.api.model.IntOrString) ParallelSuite(io.strimzi.test.annotations.ParallelSuite) ClientConfig(io.strimzi.kafka.oauth.client.ClientConfig) PodDisruptionBudget(io.fabric8.kubernetes.api.model.policy.v1.PodDisruptionBudget) Collections.singletonList(java.util.Collections.singletonList) CoreMatchers.notNullValue(org.hamcrest.CoreMatchers.notNullValue) JvmOptionsBuilder(io.strimzi.api.kafka.model.JvmOptionsBuilder) KafkaBridge(io.strimzi.api.kafka.model.KafkaBridge) JaegerTracing(io.strimzi.api.kafka.model.tracing.JaegerTracing) Map(java.util.Map) ContainerEnvVar(io.strimzi.api.kafka.model.ContainerEnvVar) ResourceUtils(io.strimzi.operator.cluster.ResourceUtils) Affinity(io.fabric8.kubernetes.api.model.Affinity) KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) LabelSelectorBuilder(io.fabric8.kubernetes.api.model.LabelSelectorBuilder) IpFamilyPolicy(io.strimzi.api.kafka.model.template.IpFamilyPolicy) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) ParallelTest(io.strimzi.test.annotations.ParallelTest) Collections.emptyList(java.util.Collections.emptyList) KafkaClientAuthenticationOAuthBuilder(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuthBuilder) EnvVarBuilder(io.fabric8.kubernetes.api.model.EnvVarBuilder) HasMetadata(io.fabric8.kubernetes.api.model.HasMetadata) KafkaBridgeHttpConfig(io.strimzi.api.kafka.model.KafkaBridgeHttpConfig) NodeSelectorTermBuilder(io.fabric8.kubernetes.api.model.NodeSelectorTermBuilder) List(java.util.List) CertSecretSourceBuilder(io.strimzi.api.kafka.model.CertSecretSourceBuilder) Labels(io.strimzi.operator.common.model.Labels) PodSecurityContextBuilder(io.fabric8.kubernetes.api.model.PodSecurityContextBuilder) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) TopologySpreadConstraintBuilder(io.fabric8.kubernetes.api.model.TopologySpreadConstraintBuilder) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) EnvVar(io.fabric8.kubernetes.api.model.EnvVar) ContainerTemplate(io.strimzi.api.kafka.model.template.ContainerTemplate) Container(io.fabric8.kubernetes.api.model.Container) ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder) KafkaClientAuthenticationTlsBuilder(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationTlsBuilder) IpFamily(io.strimzi.api.kafka.model.template.IpFamily) HashMap(java.util.HashMap) LocalObjectReference(io.fabric8.kubernetes.api.model.LocalObjectReference) OwnerReference(io.fabric8.kubernetes.api.model.OwnerReference) ArrayList(java.util.ArrayList) KafkaVersionTestUtils(io.strimzi.operator.cluster.KafkaVersionTestUtils) DeploymentStrategy(io.strimzi.api.kafka.model.template.DeploymentStrategy) TestUtils(io.strimzi.test.TestUtils) KafkaBridgeBuilder(io.strimzi.api.kafka.model.KafkaBridgeBuilder) Collections.singletonMap(java.util.Collections.singletonMap) Service(io.fabric8.kubernetes.api.model.Service) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) CoreMatchers.nullValue(org.hamcrest.CoreMatchers.nullValue) KafkaBridgeResources(io.strimzi.api.kafka.model.KafkaBridgeResources) Collections.emptyMap(java.util.Collections.emptyMap) TopologySpreadConstraint(io.fabric8.kubernetes.api.model.TopologySpreadConstraint) Toleration(io.fabric8.kubernetes.api.model.Toleration) ServerConfig(io.strimzi.kafka.oauth.server.ServerConfig) TolerationBuilder(io.fabric8.kubernetes.api.model.TolerationBuilder) AffinityBuilder(io.fabric8.kubernetes.api.model.AffinityBuilder) Reconciliation(io.strimzi.operator.common.Reconciliation) SystemPropertyBuilder(io.strimzi.api.kafka.model.SystemPropertyBuilder) ServiceAccount(io.fabric8.kubernetes.api.model.ServiceAccount) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) Collections(java.util.Collections) KeyToPath(io.fabric8.kubernetes.api.model.KeyToPath) Container(io.fabric8.kubernetes.api.model.Container) CertSecretSourceBuilder(io.strimzi.api.kafka.model.CertSecretSourceBuilder) KafkaBridge(io.strimzi.api.kafka.model.KafkaBridge) KafkaBridgeBuilder(io.strimzi.api.kafka.model.KafkaBridgeBuilder) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) ParallelTest(io.strimzi.test.annotations.ParallelTest)

Example 22 with CertSecretSource

use of io.strimzi.api.kafka.model.CertSecretSource in project strimzi-kafka-operator by strimzi.

the class UtilTest method getHashOk.

@Test
public void getHashOk() {
    String namespace = "ns";
    GenericSecretSource at = new GenericSecretSourceBuilder().withSecretName("top-secret-at").withKey("key").build();
    GenericSecretSource cs = new GenericSecretSourceBuilder().withSecretName("top-secret-cs").withKey("key").build();
    GenericSecretSource rt = new GenericSecretSourceBuilder().withSecretName("top-secret-rt").withKey("key").build();
    KafkaClientAuthentication kcu = new KafkaClientAuthenticationOAuthBuilder().withAccessToken(at).withRefreshToken(rt).withClientSecret(cs).build();
    CertSecretSource css = new CertSecretSourceBuilder().withCertificate("key").withSecretName("css-secret").build();
    Secret secret = new SecretBuilder().withData(Map.of("key", "value")).build();
    SecretOperator secretOps = mock(SecretOperator.class);
    when(secretOps.getAsync(eq(namespace), eq("top-secret-at"))).thenReturn(Future.succeededFuture(secret));
    when(secretOps.getAsync(eq(namespace), eq("top-secret-rt"))).thenReturn(Future.succeededFuture(secret));
    when(secretOps.getAsync(eq(namespace), eq("top-secret-cs"))).thenReturn(Future.succeededFuture(secret));
    when(secretOps.getAsync(eq(namespace), eq("css-secret"))).thenReturn(Future.succeededFuture(secret));
    Future<Integer> res = Util.authTlsHash(secretOps, "ns", kcu, singletonList(css));
    res.onComplete(v -> {
        assertThat(v.succeeded(), is(true));
        // we are summing "value" hash four times
        assertThat(v.result(), is("value".hashCode() * 4));
    });
}
Also used : KafkaClientAuthentication(io.strimzi.api.kafka.model.authentication.KafkaClientAuthentication) KafkaClientAuthenticationOAuthBuilder(io.strimzi.api.kafka.model.authentication.KafkaClientAuthenticationOAuthBuilder) Secret(io.fabric8.kubernetes.api.model.Secret) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) SecretOperator(io.strimzi.operator.common.operator.resource.SecretOperator) GenericSecretSource(io.strimzi.api.kafka.model.GenericSecretSource) GenericSecretSourceBuilder(io.strimzi.api.kafka.model.GenericSecretSourceBuilder) CertSecretSourceBuilder(io.strimzi.api.kafka.model.CertSecretSourceBuilder) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) Test(org.junit.jupiter.api.Test)

Example 23 with CertSecretSource

use of io.strimzi.api.kafka.model.CertSecretSource in project strimzi-kafka-operator by strimzi.

the class HttpBridgeKafkaExternalListenersST method testTlsAuthWithWeirdUsername.

@ParallelTest
void testTlsAuthWithWeirdUsername(ExtensionContext extensionContext) {
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    // Create weird named user with . and maximum of 64 chars -> TLS
    final String weirdUserName = "jjglmahyijoambryleyxjjglmahy.ijoambryleyxjjglmahyijoambryleyxasd";
    // Initialize CertSecretSource with certificate and secret names for consumer
    CertSecretSource certSecret = new CertSecretSource();
    certSecret.setCertificate("ca.crt");
    certSecret.setSecretName(KafkaResources.clusterCaCertificateSecretName(clusterName));
    KafkaBridgeSpec bridgeSpec = new KafkaBridgeSpecBuilder().withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(weirdUserName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecret).endTls().build();
    testWeirdUsername(extensionContext, weirdUserName, new KafkaListenerAuthenticationTls(), bridgeSpec, SecurityProtocol.SSL);
}
Also used : KafkaBridgeSpec(io.strimzi.api.kafka.model.KafkaBridgeSpec) KafkaBridgeSpecBuilder(io.strimzi.api.kafka.model.KafkaBridgeSpecBuilder) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) ParallelTest(io.strimzi.systemtest.annotations.ParallelTest)

Example 24 with CertSecretSource

use of io.strimzi.api.kafka.model.CertSecretSource in project strimzi-kafka-operator by strimzi.

the class MirrorMakerIsolatedST method testMirrorMakerTlsScramSha.

/**
 * Test mirroring messages by Mirror Maker over tls transport using scram-sha auth
 */
@ParallelNamespaceTest
@SuppressWarnings("checkstyle:methodlength")
void testMirrorMakerTlsScramSha(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    String kafkaClusterSourceName = clusterName + "-source";
    String kafkaClusterTargetName = clusterName + "-target";
    String kafkaUserSource = clusterName + "-my-user-source";
    String kafkaUserTarget = clusterName + "-my-user-target";
    // Deploy source kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and SCRAM-SHA authentication
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).editSpec().editKafka().withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationScramSha512()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicName).build());
    // createAndWaitForReadiness Kafka user for source cluster
    KafkaUser userSource = KafkaUserTemplates.scramShaUser(kafkaClusterSourceName, kafkaUserSource).build();
    // createAndWaitForReadiness Kafka user for target cluster
    KafkaUser userTarget = KafkaUserTemplates.scramShaUser(kafkaClusterTargetName, kafkaUserTarget).build();
    resourceManager.createResource(extensionContext, userSource);
    resourceManager.createResource(extensionContext, userTarget);
    // Initialize PasswordSecretSource to set this as PasswordSecret in Mirror Maker spec
    PasswordSecretSource passwordSecretSource = new PasswordSecretSource();
    passwordSecretSource.setSecretName(kafkaUserSource);
    passwordSecretSource.setPassword("password");
    // Initialize PasswordSecretSource to set this as PasswordSecret in Mirror Maker spec
    PasswordSecretSource passwordSecretTarget = new PasswordSecretSource();
    passwordSecretTarget.setSecretName(kafkaUserTarget);
    passwordSecretTarget.setPassword("password");
    // Initialize CertSecretSource with certificate and secret names for consumer
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for producer
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    // Deploy client
    resourceManager.createResource(extensionContext, KafkaClientsTemplates.kafkaClients(namespaceName, true, clusterName + "-" + Constants.KAFKA_CLIENTS, userSource, userTarget).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, clusterName + "-" + Constants.KAFKA_CLIENTS).get(0).getMetadata().getName();
    String baseTopic = mapWithTestTopics.get(extensionContext.getDisplayName());
    String topicTestName1 = baseTopic + "-test-1";
    String topicTestName2 = baseTopic + "-test-2";
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicTestName1).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicTestName2).build());
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicTestName1).withNamespaceName(namespaceName).withClusterName(kafkaClusterSourceName).withKafkaUsername(userSource.getMetadata().getName()).withMessageCount(messagesCount).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    // Check brokers availability
    internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTestName2).withClusterName(kafkaClusterTargetName).withKafkaUsername(userTarget.getMetadata().getName()).build();
    internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
    // Deploy Mirror Maker with TLS and ScramSha512
    resourceManager.createResource(extensionContext, KafkaMirrorMakerTemplates.kafkaMirrorMaker(clusterName, kafkaClusterSourceName, kafkaClusterTargetName, ClientUtils.generateRandomConsumerGroup(), 1, true).editSpec().editConsumer().withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserSource).withPasswordSecret(passwordSecretSource).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretSource).endTls().endConsumer().editProducer().withNewKafkaClientAuthenticationScramSha512().withUsername(kafkaUserTarget).withPasswordSecret(passwordSecretTarget).endKafkaClientAuthenticationScramSha512().withNewTls().withTrustedCertificates(certSecretTarget).endTls().endProducer().endSpec().build());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicName).withClusterName(kafkaClusterSourceName).withKafkaUsername(userSource.getMetadata().getName()).build();
    internalKafkaClient.produceAndConsumesTlsMessagesUntilBothOperationsAreSuccessful();
    InternalKafkaClient newInternalKafkaClient = internalKafkaClient.toBuilder().withClusterName(kafkaClusterTargetName).withKafkaUsername(userTarget.getMetadata().getName()).build();
    newInternalKafkaClient.consumesTlsMessagesUntilOperationIsSuccessful(internalKafkaClient.getMessageCount());
}
Also used : KafkaListenerAuthenticationScramSha512(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationScramSha512) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) ResourceRequirementsBuilder(io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder) PasswordSecretSource(io.strimzi.api.kafka.model.PasswordSecretSource) InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaUser(io.strimzi.api.kafka.model.KafkaUser) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Example 25 with CertSecretSource

use of io.strimzi.api.kafka.model.CertSecretSource in project strimzi-kafka-operator by strimzi.

the class MirrorMaker2IsolatedST method testKMM2RollAfterSecretsCertsUpdateTLS.

@ParallelNamespaceTest
@SuppressWarnings({ "checkstyle:MethodLength" })
void testKMM2RollAfterSecretsCertsUpdateTLS(ExtensionContext extensionContext) {
    TestStorage testStorage = new TestStorage(extensionContext);
    String kafkaClusterSourceName = testStorage.getClusterName() + "-source";
    String kafkaClusterTargetName = testStorage.getClusterName() + "-target";
    String topicSourceNameA = MIRRORMAKER2_TOPIC_NAME + "-a-" + rng.nextInt(Integer.MAX_VALUE);
    String topicSourceNameB = MIRRORMAKER2_TOPIC_NAME + "-b-" + rng.nextInt(Integer.MAX_VALUE);
    String topicTargetNameA = kafkaClusterSourceName + "." + topicSourceNameA;
    String topicTargetNameB = kafkaClusterSourceName + "." + topicSourceNameB;
    String kafkaUserSourceName = testStorage.getClusterName() + "-my-user-source";
    String kafkaUserTargetName = testStorage.getClusterName() + "-my-user-target";
    String kafkaTlsScramListenerName = "tlsscram";
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    // Deploy source kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 3, 3).editSpec().editKafka().addToConfig("min.insync.replicas", 1).withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy target kafka with tls listener and mutual tls auth
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 3, 3).editSpec().editKafka().addToConfig("min.insync.replicas", 1).withListeners(new GenericKafkaListenerBuilder().withName(Constants.TLS_LISTENER_DEFAULT_NAME).withPort(9093).withType(KafkaListenerType.INTERNAL).withTls(true).withAuth(new KafkaListenerAuthenticationTls()).build()).endKafka().endSpec().build());
    // Deploy topic
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicSourceNameA, 3).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicSourceNameB, 3).build());
    // Create Kafka user
    KafkaUser userSource = KafkaUserTemplates.tlsUser(kafkaClusterSourceName, kafkaUserSourceName).build();
    KafkaUser userTarget = KafkaUserTemplates.tlsUser(kafkaClusterTargetName, kafkaUserTargetName).build();
    resourceManager.createResource(extensionContext, userSource);
    resourceManager.createResource(extensionContext, userTarget);
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(testStorage.getNamespaceName(), true, kafkaClientsName, userSource, userTarget).build());
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(testStorage.getNamespaceName(), kafkaClientsName).get(0).getMetadata().getName();
    String baseTopic = mapWithTestTopics.get(extensionContext.getDisplayName());
    String topicTestName1 = baseTopic + "-test-1";
    String topicTestName2 = baseTopic + "-test-2";
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterSourceName, topicTestName1).build());
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(kafkaClusterTargetName, topicTestName2).build());
    InternalKafkaClient internalKafkaClient = new InternalKafkaClient.Builder().withUsingPodName(kafkaClientsPodName).withTopicName(topicTestName1).withNamespaceName(testStorage.getNamespaceName()).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withMessageCount(messagesCount).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    internalKafkaClient = internalKafkaClient.toBuilder().withClusterName(kafkaClusterTargetName).withTopicName(topicTestName2).withKafkaUsername(kafkaUserTargetName).build();
    internalKafkaClient.checkProducedAndConsumedMessages(internalKafkaClient.sendMessagesTls(), internalKafkaClient.receiveMessagesTls());
    // Initialize CertSecretSource with certificate and secret names for source
    CertSecretSource certSecretSource = new CertSecretSource();
    certSecretSource.setCertificate("ca.crt");
    certSecretSource.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName));
    // Initialize CertSecretSource with certificate and secret names for target
    CertSecretSource certSecretTarget = new CertSecretSource();
    certSecretTarget.setCertificate("ca.crt");
    certSecretTarget.setSecretName(KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName));
    // Deploy Mirror Maker 2.0 with tls listener and mutual tls auth
    KafkaMirrorMaker2ClusterSpec sourceClusterWithTlsAuth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaClusterSourceName).withBootstrapServers(KafkaResources.tlsBootstrapAddress(kafkaClusterSourceName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserSourceName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretSource).endTls().build();
    KafkaMirrorMaker2ClusterSpec targetClusterWithTlsAuth = new KafkaMirrorMaker2ClusterSpecBuilder().withAlias(kafkaClusterTargetName).withBootstrapServers(KafkaResources.tlsBootstrapAddress(kafkaClusterTargetName)).withNewKafkaClientAuthenticationTls().withNewCertificateAndKey().withSecretName(kafkaUserTargetName).withCertificate("user.crt").withKey("user.key").endCertificateAndKey().endKafkaClientAuthenticationTls().withNewTls().withTrustedCertificates(certSecretTarget).endTls().addToConfig("config.storage.replication.factor", -1).addToConfig("offset.storage.replication.factor", -1).addToConfig("status.storage.replication.factor", -1).build();
    resourceManager.createResource(extensionContext, KafkaMirrorMaker2Templates.kafkaMirrorMaker2(testStorage.getClusterName(), kafkaClusterTargetName, kafkaClusterSourceName, 1, true).editSpec().withClusters(targetClusterWithTlsAuth, sourceClusterWithTlsAuth).editFirstMirror().withTopicsPattern(MIRRORMAKER2_TOPIC_NAME + ".*").editSourceConnector().addToConfig("refresh.topics.interval.seconds", 1).endSourceConnector().endMirror().endSpec().build());
    String mm2DeploymentName = KafkaMirrorMaker2Resources.deploymentName(testStorage.getClusterName());
    Map<String, String> mmSnapshot = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), mm2DeploymentName);
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceNameA).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    int sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetNameA).withClusterName(kafkaClusterTargetName).withKafkaUsername(kafkaUserTargetName).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
    LabelSelector zkSourceSelector = KafkaResource.getLabelSelector(kafkaClusterSourceName, KafkaResources.zookeeperStatefulSetName(kafkaClusterSourceName));
    LabelSelector kafkaSourceSelector = KafkaResource.getLabelSelector(kafkaClusterSourceName, KafkaResources.kafkaStatefulSetName(kafkaClusterSourceName));
    LabelSelector zkTargetSelector = KafkaResource.getLabelSelector(kafkaClusterTargetName, KafkaResources.zookeeperStatefulSetName(kafkaClusterTargetName));
    LabelSelector kafkaTargetSelector = KafkaResource.getLabelSelector(kafkaClusterTargetName, KafkaResources.kafkaStatefulSetName(kafkaClusterTargetName));
    Map<String, String> kafkaSourcePods = PodUtils.podSnapshot(testStorage.getNamespaceName(), kafkaSourceSelector);
    Map<String, String> zkSourcePods = PodUtils.podSnapshot(testStorage.getNamespaceName(), zkSourceSelector);
    Map<String, String> eoSourcePods = DeploymentUtils.depSnapshot(KafkaResources.entityOperatorDeploymentName(kafkaClusterSourceName));
    Map<String, String> kafkaTargetPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), kafkaTargetSelector);
    Map<String, String> zkTargetPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), zkTargetSelector);
    Map<String, String> eoTargetPods = DeploymentUtils.depSnapshot(KafkaResources.entityOperatorDeploymentName(kafkaClusterTargetName));
    LOGGER.info("Renew Clients CA secret for Source cluster via annotation");
    String sourceClientsCaSecretName = KafkaResources.clientsCaCertificateSecretName(kafkaClusterSourceName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), sourceClientsCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    kafkaSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaSourceSelector, 3, kafkaSourcePods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Renew Clients CA secret for Target cluster via annotation");
    String targetClientsCaSecretName = KafkaResources.clientsCaCertificateSecretName(kafkaClusterTargetName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), targetClientsCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    kafkaTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaTargetSelector, 3, kafkaTargetPods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Send and receive messages after clients certs were removed");
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceNameA).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetNameA).withClusterName(kafkaClusterTargetName).withKafkaUsername(kafkaUserTargetName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
    LOGGER.info("Renew Cluster CA secret for Source clusters via annotation");
    String sourceClusterCaSecretName = KafkaResources.clusterCaCertificateSecretName(kafkaClusterSourceName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), sourceClusterCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    zkSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), zkSourceSelector, 3, zkSourcePods);
    kafkaSourcePods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaSourceSelector, 3, kafkaSourcePods);
    eoSourcePods = DeploymentUtils.waitTillDepHasRolled(KafkaResources.entityOperatorDeploymentName(kafkaClusterSourceName), 1, eoSourcePods);
    mmSnapshot = DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Renew Cluster CA secret for Target clusters via annotation");
    String targetClusterCaSecretName = KafkaResources.clusterCaCertificateSecretName(kafkaClusterTargetName);
    SecretUtils.annotateSecret(testStorage.getNamespaceName(), targetClusterCaSecretName, Ca.ANNO_STRIMZI_IO_FORCE_RENEW, "true");
    zkTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), zkTargetSelector, 3, zkTargetPods);
    kafkaTargetPods = RollingUpdateUtils.waitTillComponentHasRolledAndPodsReady(testStorage.getNamespaceName(), kafkaTargetSelector, 3, kafkaTargetPods);
    eoTargetPods = DeploymentUtils.waitTillDepHasRolled(KafkaResources.entityOperatorDeploymentName(kafkaClusterTargetName), 1, eoTargetPods);
    DeploymentUtils.waitTillDepHasRolled(testStorage.getNamespaceName(), mm2DeploymentName, 1, mmSnapshot);
    LOGGER.info("Send and receive messages after clients certs were removed");
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicSourceNameB).withClusterName(kafkaClusterSourceName).withKafkaUsername(kafkaUserSourceName).withListenerName(Constants.TLS_LISTENER_DEFAULT_NAME).build();
    sent = internalKafkaClient.sendMessagesTls();
    internalKafkaClient = internalKafkaClient.toBuilder().withTopicName(topicTargetNameB).withClusterName(kafkaClusterTargetName).withKafkaUsername(kafkaUserTargetName).withConsumerGroupName(ClientUtils.generateRandomConsumerGroup()).build();
    LOGGER.info("Consumer in target cluster and topic should receive {} messages", messagesCount);
    internalKafkaClient.checkProducedAndConsumedMessages(sent, internalKafkaClient.receiveMessagesTls());
    LOGGER.info("Messages successfully mirrored");
}
Also used : GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) JobBuilder(io.fabric8.kubernetes.api.model.batch.v1.JobBuilder) KafkaClientsBuilder(io.strimzi.systemtest.kafkaclients.internalClients.KafkaClientsBuilder) KafkaMirrorMaker2ClusterSpecBuilder(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpecBuilder) SecretBuilder(io.fabric8.kubernetes.api.model.SecretBuilder) LabelSelector(io.fabric8.kubernetes.api.model.LabelSelector) Matchers.containsString(org.hamcrest.Matchers.containsString) KafkaListenerAuthenticationTls(io.strimzi.api.kafka.model.listener.KafkaListenerAuthenticationTls) GenericKafkaListenerBuilder(io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder) KafkaMirrorMaker2ClusterSpecBuilder(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpecBuilder) KafkaMirrorMaker2ClusterSpec(io.strimzi.api.kafka.model.KafkaMirrorMaker2ClusterSpec) InternalKafkaClient(io.strimzi.systemtest.kafkaclients.clients.InternalKafkaClient) TestStorage(io.strimzi.systemtest.storage.TestStorage) CertSecretSource(io.strimzi.api.kafka.model.CertSecretSource) KafkaUser(io.strimzi.api.kafka.model.KafkaUser) ParallelNamespaceTest(io.strimzi.systemtest.annotations.ParallelNamespaceTest)

Aggregations

CertSecretSource (io.strimzi.api.kafka.model.CertSecretSource)70 CertSecretSourceBuilder (io.strimzi.api.kafka.model.CertSecretSourceBuilder)30 GenericKafkaListenerBuilder (io.strimzi.api.kafka.model.listener.arraylistener.GenericKafkaListenerBuilder)26 ParallelTest (io.strimzi.test.annotations.ParallelTest)24 ArrayList (java.util.ArrayList)24 ResourceRequirementsBuilder (io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder)22 ServiceAccount (io.fabric8.kubernetes.api.model.ServiceAccount)22 Reconciliation (io.strimzi.operator.common.Reconciliation)22 Collections (java.util.Collections)22 List (java.util.List)22 ConfigMap (io.fabric8.kubernetes.api.model.ConfigMap)20 EnvVar (io.fabric8.kubernetes.api.model.EnvVar)20 ContainerEnvVar (io.strimzi.api.kafka.model.ContainerEnvVar)20 HashMap (java.util.HashMap)20 Map (java.util.Map)20 Container (io.fabric8.kubernetes.api.model.Container)18 HasMetadata (io.fabric8.kubernetes.api.model.HasMetadata)18 IntOrString (io.fabric8.kubernetes.api.model.IntOrString)18 LocalObjectReference (io.fabric8.kubernetes.api.model.LocalObjectReference)18 OwnerReference (io.fabric8.kubernetes.api.model.OwnerReference)18