Examples with KafkaMirrorMaker io.strimzi.api.kafka.model.KafkaMirrorMaker used on opensource projects

Example 21 with KafkaMirrorMaker

use of io.strimzi.api.kafka.model.KafkaMirrorMaker in project strimzi by strimzi.

the class KafkaMirrorMakerAssemblyOperatorTest method testReconcile.

@Test
public void testReconcile(VertxTestContext context) {
    ResourceOperatorSupplier supplier = ResourceUtils.supplierWithMocks(true);
    CrdOperator mockMirrorOps = supplier.mirrorMakerOperator;
    DeploymentOperator mockDcOps = supplier.deploymentOperations;
    SecretOperator mockSecretOps = supplier.secretOperations;
    String kmmNamespace = "test";
    KafkaMirrorMakerConsumerSpec consumer = new KafkaMirrorMakerConsumerSpecBuilder().withBootstrapServers(consumerBootstrapServers).withGroupId(groupId).withNumStreams(numStreams).build();
    KafkaMirrorMakerProducerSpec producer = new KafkaMirrorMakerProducerSpecBuilder().withBootstrapServers(producerBootstrapServers).build();
    Map<String, Object> metricsCm = new HashMap<>();
    metricsCm.put("foo", "bar");
    KafkaMirrorMaker foo = ResourceUtils.createKafkaMirrorMaker(kmmNamespace, "foo", image, producer, consumer, include);
    KafkaMirrorMaker bar = ResourceUtils.createKafkaMirrorMaker(kmmNamespace, "bar", image, producer, consumer, include);
    when(mockMirrorOps.listAsync(eq(kmmNamespace), any(Optional.class))).thenReturn(Future.succeededFuture(asList(foo, bar)));
    // when requested ConfigMap for a specific Kafka Mirror Maker cluster
    when(mockMirrorOps.get(eq(kmmNamespace), eq("foo"))).thenReturn(foo);
    when(mockMirrorOps.get(eq(kmmNamespace), eq("bar"))).thenReturn(bar);
    when(mockMirrorOps.getAsync(anyString(), anyString())).thenReturn(Future.succeededFuture());
    // providing the list of ALL Deployments for all the Kafka Mirror Maker clusters
    Labels newLabels = Labels.forStrimziKind(KafkaMirrorMaker.RESOURCE_KIND);
    when(mockDcOps.list(eq(kmmNamespace), eq(newLabels))).thenReturn(asList(KafkaMirrorMakerCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, bar, VERSIONS).generateDeployment(new HashMap<String, String>(), true, null, null)));
    // providing the list Deployments for already "existing" Kafka Mirror Maker clusters
    Labels barLabels = Labels.forStrimziCluster("bar");
    when(mockDcOps.list(eq(kmmNamespace), eq(barLabels))).thenReturn(asList(KafkaMirrorMakerCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, bar, VERSIONS).generateDeployment(new HashMap<String, String>(), true, null, null)));
    when(mockDcOps.readiness(any(), anyString(), anyString(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockDcOps.waitForObserved(any(), anyString(), anyString(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockSecretOps.reconcile(any(), eq(kmmNamespace), any(), any())).thenReturn(Future.succeededFuture());
    Set<String> createdOrUpdated = new CopyOnWriteArraySet<>();
    Checkpoint createdOrUpdateAsync = context.checkpoint(2);
    KafkaMirrorMakerAssemblyOperator ops = new KafkaMirrorMakerAssemblyOperator(vertx, new PlatformFeaturesAvailability(true, kubernetesVersion), new MockCertManager(), new PasswordGenerator(10, "a", "a"), supplier, ResourceUtils.dummyClusterOperatorConfig(VERSIONS)) {

        @Override
        public Future<KafkaMirrorMakerStatus> createOrUpdate(Reconciliation reconciliation, KafkaMirrorMaker kafkaMirrorMakerAssembly) {
            createdOrUpdated.add(kafkaMirrorMakerAssembly.getMetadata().getName());
            createdOrUpdateAsync.flag();
            return Future.succeededFuture();
        }
    };
    Checkpoint async = context.checkpoint();
    // Now try to reconcile all the Kafka Mirror Maker clusters
    ops.reconcileAll("test", kmmNamespace, context.succeeding(v -> context.verify(() -> {
        assertThat(createdOrUpdated, is(new HashSet(asList("foo", "bar"))));
        async.flag();
    })));
}

Example 22 with KafkaMirrorMaker

use of io.strimzi.api.kafka.model.KafkaMirrorMaker in project strimzi by strimzi.

the class KafkaMirrorMakerAssemblyOperatorTest method testUpdateClusterFailure.

@Test
public void testUpdateClusterFailure(VertxTestContext context) {
    ResourceOperatorSupplier supplier = ResourceUtils.supplierWithMocks(true);
    CrdOperator mockMirrorOps = supplier.mirrorMakerOperator;
    DeploymentOperator mockDcOps = supplier.deploymentOperations;
    PodDisruptionBudgetOperator mockPdbOps = supplier.podDisruptionBudgetOperator;
    ConfigMapOperator mockCmOps = supplier.configMapOperations;
    String kmmName = "foo";
    String kmmNamespace = "test";
    KafkaMirrorMakerConsumerSpec consumer = new KafkaMirrorMakerConsumerSpecBuilder().withBootstrapServers(consumerBootstrapServers).withGroupId(groupId).withNumStreams(numStreams).build();
    KafkaMirrorMakerProducerSpec producer = new KafkaMirrorMakerProducerSpecBuilder().withBootstrapServers(producerBootstrapServers).build();
    Map<String, Object> metricsCm = new HashMap<>();
    metricsCm.put("foo", "bar");
    KafkaMirrorMaker kmm = ResourceUtils.createKafkaMirrorMaker(kmmNamespace, kmmName, image, producer, consumer, include);
    KafkaMirrorMakerCluster mirror = KafkaMirrorMakerCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kmm, VERSIONS);
    // Change the image to generate some diff
    kmm.getSpec().setImage("some/different:image");
    when(mockMirrorOps.get(kmmNamespace, kmmName)).thenReturn(kmm);
    when(mockDcOps.get(kmmNamespace, mirror.getName())).thenReturn(mirror.generateDeployment(new HashMap<String, String>(), true, null, null));
    when(mockDcOps.readiness(any(), eq(kmmNamespace), eq(mirror.getName()), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockDcOps.waitForObserved(any(), anyString(), anyString(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    ArgumentCaptor<String> dcNamespaceCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> dcNameCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<Deployment> dcCaptor = ArgumentCaptor.forClass(Deployment.class);
    when(mockDcOps.reconcile(any(), dcNamespaceCaptor.capture(), dcNameCaptor.capture(), dcCaptor.capture())).thenReturn(Future.failedFuture("Failed"));
    ArgumentCaptor<String> dcScaleUpNamespaceCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> dcScaleUpNameCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<Integer> dcScaleUpReplicasCaptor = ArgumentCaptor.forClass(Integer.class);
    when(mockDcOps.scaleUp(any(), dcScaleUpNamespaceCaptor.capture(), dcScaleUpNameCaptor.capture(), dcScaleUpReplicasCaptor.capture())).thenReturn(Future.succeededFuture());
    ArgumentCaptor<String> dcScaleDownNamespaceCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<String> dcScaleDownNameCaptor = ArgumentCaptor.forClass(String.class);
    ArgumentCaptor<Integer> dcScaleDownReplicasCaptor = ArgumentCaptor.forClass(Integer.class);
    when(mockDcOps.scaleDown(any(), dcScaleDownNamespaceCaptor.capture(), dcScaleDownNameCaptor.capture(), dcScaleDownReplicasCaptor.capture())).thenReturn(Future.succeededFuture());
    when(mockPdbOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture());
    when(mockMirrorOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture(ReconcileResult.created(new KafkaMirrorMaker())));
    when(mockMirrorOps.getAsync(anyString(), anyString())).thenReturn(Future.succeededFuture(kmm));
    when(mockMirrorOps.updateStatusAsync(any(), any(KafkaMirrorMaker.class))).thenReturn(Future.succeededFuture());
    when(mockCmOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture(ReconcileResult.created(new ConfigMap())));
    KafkaMirrorMakerAssemblyOperator ops = new KafkaMirrorMakerAssemblyOperator(vertx, new PlatformFeaturesAvailability(true, kubernetesVersion), new MockCertManager(), new PasswordGenerator(10, "a", "a"), supplier, ResourceUtils.dummyClusterOperatorConfig(VERSIONS));
    Checkpoint async = context.checkpoint();
    ops.createOrUpdate(new Reconciliation("test-trigger", KafkaMirrorMaker.RESOURCE_KIND, kmmNamespace, kmmName), kmm).onComplete(context.failing(v -> context.verify(() -> async.flag())));
}

Example 23 with KafkaMirrorMaker

use of io.strimzi.api.kafka.model.KafkaMirrorMaker in project strimzi by strimzi.

the class KafkaMirrorMakerAssemblyOperatorTest method testUpdateClusterScaleDown.

@Test
public void testUpdateClusterScaleDown(VertxTestContext context) {
    int scaleTo = 2;
    ResourceOperatorSupplier supplier = ResourceUtils.supplierWithMocks(true);
    CrdOperator mockMirrorOps = supplier.mirrorMakerOperator;
    DeploymentOperator mockDcOps = supplier.deploymentOperations;
    PodDisruptionBudgetOperator mockPdbOps = supplier.podDisruptionBudgetOperator;
    ConfigMapOperator mockCmOps = supplier.configMapOperations;
    String kmmName = "foo";
    String kmmNamespace = "test";
    KafkaMirrorMakerConsumerSpec consumer = new KafkaMirrorMakerConsumerSpecBuilder().withBootstrapServers(consumerBootstrapServers).withGroupId(groupId).withNumStreams(numStreams).build();
    KafkaMirrorMakerProducerSpec producer = new KafkaMirrorMakerProducerSpecBuilder().withBootstrapServers(producerBootstrapServers).build();
    Map<String, Object> metricsCm = new HashMap<>();
    metricsCm.put("foo", "bar");
    KafkaMirrorMaker kmm = ResourceUtils.createKafkaMirrorMaker(kmmNamespace, kmmName, image, producer, consumer, include);
    KafkaMirrorMakerCluster mirror = KafkaMirrorMakerCluster.fromCrd(Reconciliation.DUMMY_RECONCILIATION, kmm, VERSIONS);
    // Change replicas to create ScaleDown
    kmm.getSpec().setReplicas(scaleTo);
    when(mockMirrorOps.get(kmmNamespace, kmmName)).thenReturn(kmm);
    when(mockMirrorOps.getAsync(anyString(), anyString())).thenReturn(Future.succeededFuture(kmm));
    when(mockMirrorOps.updateStatusAsync(any(), any(KafkaMirrorMaker.class))).thenReturn(Future.succeededFuture());
    when(mockDcOps.get(kmmNamespace, mirror.getName())).thenReturn(mirror.generateDeployment(new HashMap<String, String>(), true, null, null));
    when(mockDcOps.readiness(any(), eq(kmmNamespace), eq(mirror.getName()), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockDcOps.waitForObserved(any(), anyString(), anyString(), anyLong(), anyLong())).thenReturn(Future.succeededFuture());
    when(mockDcOps.reconcile(any(), eq(kmmNamespace), any(), any())).thenReturn(Future.succeededFuture());
    doAnswer(i -> Future.succeededFuture(scaleTo)).when(mockDcOps).scaleUp(any(), eq(kmmNamespace), eq(mirror.getName()), eq(scaleTo));
    doAnswer(i -> Future.succeededFuture(scaleTo)).when(mockDcOps).scaleDown(any(), eq(kmmNamespace), eq(mirror.getName()), eq(scaleTo));
    when(mockMirrorOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture(ReconcileResult.created(new KafkaMirrorMaker())));
    when(mockCmOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture(ReconcileResult.created(new ConfigMap())));
    when(mockPdbOps.reconcile(any(), anyString(), any(), any())).thenReturn(Future.succeededFuture());
    KafkaMirrorMakerAssemblyOperator ops = new KafkaMirrorMakerAssemblyOperator(vertx, new PlatformFeaturesAvailability(true, kubernetesVersion), new MockCertManager(), new PasswordGenerator(10, "a", "a"), supplier, ResourceUtils.dummyClusterOperatorConfig(VERSIONS));
    Checkpoint async = context.checkpoint();
    ops.createOrUpdate(new Reconciliation("test-trigger", KafkaMirrorMaker.RESOURCE_KIND, kmmNamespace, kmmName), kmm).onComplete(context.succeeding(v -> context.verify(() -> {
        verify(mockDcOps).scaleUp(any(), eq(kmmNamespace), eq(mirror.getName()), eq(scaleTo));
        async.flag();
    })));
}

Example 24 with KafkaMirrorMaker

use of io.strimzi.api.kafka.model.KafkaMirrorMaker in project strimzi by strimzi.

the class MirrorMakerIsolatedST method testConfigureDeploymentStrategy.

@ParallelNamespaceTest
void testConfigureDeploymentStrategy(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(INFRA_NAMESPACE, extensionContext);
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String kafkaClusterSourceName = clusterName + "-source";
    String kafkaClusterTargetName = clusterName + "-target";
    // Deploy source kafka
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterSourceName, 1, 1).build());
    // Deploy target kafka
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(kafkaClusterTargetName, 1, 1).build());
    resourceManager.createResource(extensionContext, KafkaMirrorMakerTemplates.kafkaMirrorMaker(clusterName, kafkaClusterTargetName, kafkaClusterSourceName, ClientUtils.generateRandomConsumerGroup(), 1, false).editSpec().editOrNewTemplate().editOrNewDeployment().withDeploymentStrategy(DeploymentStrategy.RECREATE).endDeployment().endTemplate().endSpec().build());
    String mmDepName = KafkaMirrorMakerResources.deploymentName(clusterName);
    LOGGER.info("Adding label to MirrorMaker resource, the CR should be recreateAndWaitForReadinessd");
    KafkaMirrorMakerResource.replaceMirrorMakerResourceInSpecificNamespace(clusterName, mm -> mm.getMetadata().setLabels(Collections.singletonMap("some", "label")), namespaceName);
    DeploymentUtils.waitForDeploymentAndPodsReady(namespaceName, mmDepName, 1);
    KafkaMirrorMaker kmm = KafkaMirrorMakerResource.kafkaMirrorMakerClient().inNamespace(namespaceName).withName(clusterName).get();
    LOGGER.info("Checking that observed gen. is still on 1 (recreation) and new label is present");
    assertThat(kmm.getStatus().getObservedGeneration(), is(1L));
    assertThat(kmm.getMetadata().getLabels().toString(), Matchers.containsString("some=label"));
    assertThat(kmm.getSpec().getTemplate().getDeployment().getDeploymentStrategy(), is(DeploymentStrategy.RECREATE));
    LOGGER.info("Changing deployment strategy to {}", DeploymentStrategy.ROLLING_UPDATE);
    KafkaMirrorMakerResource.replaceMirrorMakerResourceInSpecificNamespace(clusterName, mm -> mm.getSpec().getTemplate().getDeployment().setDeploymentStrategy(DeploymentStrategy.ROLLING_UPDATE), namespaceName);
    KafkaMirrorMakerUtils.waitForKafkaMirrorMakerReady(namespaceName, clusterName);
    LOGGER.info("Adding another label to MirrorMaker resource, pods should be rolled");
    KafkaMirrorMakerResource.replaceMirrorMakerResourceInSpecificNamespace(clusterName, mm -> mm.getMetadata().getLabels().put("another", "label"), namespaceName);
    DeploymentUtils.waitForDeploymentAndPodsReady(namespaceName, mmDepName, 1);
    LOGGER.info("Checking that observed gen. higher (rolling update) and label is changed");
    kmm = KafkaMirrorMakerResource.kafkaMirrorMakerClient().inNamespace(namespaceName).withName(clusterName).get();
    assertThat(kmm.getStatus().getObservedGeneration(), is(2L));
    assertThat(kmm.getMetadata().getLabels().toString(), Matchers.containsString("another=label"));
    assertThat(kmm.getSpec().getTemplate().getDeployment().getDeploymentStrategy(), is(DeploymentStrategy.ROLLING_UPDATE));
}

Example 25 with KafkaMirrorMaker

use of io.strimzi.api.kafka.model.KafkaMirrorMaker in project strimzi by strimzi.

the class KafkaMirrorMakerAssemblyOperator method createOrUpdate.

@Override
protected Future<KafkaMirrorMakerStatus> createOrUpdate(Reconciliation reconciliation, KafkaMirrorMaker assemblyResource) {
    String namespace = reconciliation.namespace();
    KafkaMirrorMakerCluster mirror;
    KafkaMirrorMakerStatus kafkaMirrorMakerStatus = new KafkaMirrorMakerStatus();
    try {
        mirror = KafkaMirrorMakerCluster.fromCrd(reconciliation, assemblyResource, versions);
    } catch (Exception e) {
        LOGGER.warnCr(reconciliation, e);
        StatusUtils.setStatusConditionAndObservedGeneration(assemblyResource, kafkaMirrorMakerStatus, Future.failedFuture(e));
        return Future.failedFuture(new ReconciliationException(kafkaMirrorMakerStatus, e));
    }
    Map<String, String> annotations = new HashMap<>(1);
    KafkaClientAuthentication authConsumer = assemblyResource.getSpec().getConsumer().getAuthentication();
    List<CertSecretSource> trustedCertificatesConsumer = assemblyResource.getSpec().getConsumer().getTls() == null ? Collections.emptyList() : assemblyResource.getSpec().getConsumer().getTls().getTrustedCertificates();
    KafkaClientAuthentication authProducer = assemblyResource.getSpec().getProducer().getAuthentication();
    List<CertSecretSource> trustedCertificatesProducer = assemblyResource.getSpec().getProducer().getTls() == null ? Collections.emptyList() : assemblyResource.getSpec().getProducer().getTls().getTrustedCertificates();
    Promise<KafkaMirrorMakerStatus> createOrUpdatePromise = Promise.promise();
    boolean mirrorHasZeroReplicas = mirror.getReplicas() == 0;
    LOGGER.debugCr(reconciliation, "Updating Kafka Mirror Maker cluster");
    mirrorMakerServiceAccount(reconciliation, namespace, mirror).compose(i -> deploymentOperations.scaleDown(reconciliation, namespace, mirror.getName(), mirror.getReplicas())).compose(i -> Util.metricsAndLogging(reconciliation, configMapOperations, namespace, mirror.getLogging(), mirror.getMetricsConfigInCm())).compose(metricsAndLoggingCm -> {
        ConfigMap logAndMetricsConfigMap = mirror.generateMetricsAndLogConfigMap(metricsAndLoggingCm);
        annotations.put(Annotations.STRIMZI_LOGGING_ANNOTATION, logAndMetricsConfigMap.getData().get(mirror.ANCILLARY_CM_KEY_LOG_CONFIG));
        return configMapOperations.reconcile(reconciliation, namespace, mirror.getAncillaryConfigMapName(), logAndMetricsConfigMap);
    }).compose(i -> pfa.hasPodDisruptionBudgetV1() ? podDisruptionBudgetOperator.reconcile(reconciliation, namespace, mirror.getName(), mirror.generatePodDisruptionBudget()) : Future.succeededFuture()).compose(i -> !pfa.hasPodDisruptionBudgetV1() ? podDisruptionBudgetV1Beta1Operator.reconcile(reconciliation, namespace, mirror.getName(), mirror.generatePodDisruptionBudgetV1Beta1()) : Future.succeededFuture()).compose(i -> CompositeFuture.join(Util.authTlsHash(secretOperations, namespace, authConsumer, trustedCertificatesConsumer), Util.authTlsHash(secretOperations, namespace, authProducer, trustedCertificatesProducer))).compose(hashFut -> {
        if (hashFut != null) {
            annotations.put(Annotations.ANNO_STRIMZI_AUTH_HASH, Integer.toString((int) hashFut.resultAt(0) + (int) hashFut.resultAt(1)));
        }
        return Future.succeededFuture();
    }).compose(i -> deploymentOperations.reconcile(reconciliation, namespace, mirror.getName(), mirror.generateDeployment(annotations, pfa.isOpenshift(), imagePullPolicy, imagePullSecrets))).compose(i -> deploymentOperations.scaleUp(reconciliation, namespace, mirror.getName(), mirror.getReplicas())).compose(i -> deploymentOperations.waitForObserved(reconciliation, namespace, mirror.getName(), 1_000, operationTimeoutMs)).compose(i -> mirrorHasZeroReplicas ? Future.succeededFuture() : deploymentOperations.readiness(reconciliation, namespace, mirror.getName(), 1_000, operationTimeoutMs)).onComplete(reconciliationResult -> {
        StatusUtils.setStatusConditionAndObservedGeneration(assemblyResource, kafkaMirrorMakerStatus, reconciliationResult);
        kafkaMirrorMakerStatus.setReplicas(mirror.getReplicas());
        kafkaMirrorMakerStatus.setLabelSelector(mirror.getSelectorLabels().toSelectorString());
        if (reconciliationResult.succeeded()) {
            createOrUpdatePromise.complete(kafkaMirrorMakerStatus);
        } else {
            createOrUpdatePromise.fail(new ReconciliationException(kafkaMirrorMakerStatus, reconciliationResult.cause()));
        }
    });
    return createOrUpdatePromise.future();
}