Examples with KafkaVersion io.strimzi.operator.cluster.model.KafkaVersion used on opensource projects

Example 6 with KafkaVersion

use of io.strimzi.operator.cluster.model.KafkaVersion in project strimzi-kafka-operator by strimzi.

the class KafkaUpgradeDowngradeMockTest method testUpgradeWithoutAnyVersionInPodsOrStsFails.

// Tests upgrade when Kafka StatefulSet and/or Pods existing but without any of the version annotations. This
// indicates that the Statefulset / Pods were not using the current or recent Strimzi version and since we do not
// know the version, we should wail.
@Test
public void testUpgradeWithoutAnyVersionInPodsOrStsFails(VertxTestContext context) {
    KafkaVersion unsupported = VERSIONS.version("2.1.0");
    Kafka initialKafka = kafkaWithVersions(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion());
    Kafka updatedKafka = kafkaWithVersions(KafkaVersionTestUtils.LATEST_KAFKA_VERSION);
    Checkpoint reconciliation = context.checkpoint();
    initialize(context, initialKafka).onComplete(context.succeeding(v -> {
        context.verify(() -> {
            assertVersionsInStatefulSet(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion(), KafkaVersionTestUtils.LATEST_KAFKA_IMAGE);
        });
    })).compose(v -> {
        StatefulSet sts = client.apps().statefulSets().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka").get();
        StatefulSet modifiedSts = new StatefulSetBuilder(sts).editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).endMetadata().editSpec().editTemplate().editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_LOG_MESSAGE_FORMAT_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_INTER_BROKER_PROTOCOL_VERSION).endMetadata().endTemplate().endSpec().build();
        client.apps().statefulSets().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka").createOrReplace(modifiedSts);
        for (int i = 0; i < 3; i++) {
            Pod pod = client.pods().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka-" + i).get();
            Pod modifiedPod = new PodBuilder(pod).editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_LOG_MESSAGE_FORMAT_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_INTER_BROKER_PROTOCOL_VERSION).endMetadata().build();
            client.pods().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka-" + i).createOrReplace(modifiedPod);
        }
        return Future.succeededFuture();
    }).compose(v -> operator.createOrUpdate(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME), updatedKafka)).onComplete(context.failing(v -> context.verify(() -> {
        assertThat(v.getMessage(), stringContainsInOrder("Kafka Pods or StatefulSet exist, but do not contain the strimzi.io/kafka-version annotation to detect their version. Kafka upgrade cannot be detected."));
        reconciliation.flag();
    })));
}

Example 7 with KafkaVersion

use of io.strimzi.operator.cluster.model.KafkaVersion in project strimzi by strimzi.

the class KafkaBrokerConfigurationDiff method diff.

/**
 * Computes diff between two maps. Entries in IGNORABLE_PROPERTIES are skipped
 * @param brokerId id of compared broker
 * @param desired desired configuration, may be null if the related ConfigMap does not exist yet or no changes are required
 * @param brokerConfigs current configuration
 * @param configModel default configuration for {@code kafkaVersion} of broker
 * @return Collection of AlterConfigOp containing all entries which were changed from current in desired configuration
 */
private Collection<AlterConfigOp> diff(int brokerId, String desired, Config brokerConfigs, Map<String, ConfigModel> configModel) {
    if (brokerConfigs == null || desired == null) {
        return Collections.emptyList();
    }
    Map<String, String> currentMap;
    Collection<AlterConfigOp> updatedCE = new ArrayList<>();
    currentMap = brokerConfigs.entries().stream().collect(Collectors.toMap(ConfigEntry::name, configEntry -> configEntry.value() == null ? "null" : configEntry.value()));
    OrderedProperties orderedProperties = new OrderedProperties();
    orderedProperties.addStringPairs(desired);
    Map<String, String> desiredMap = orderedProperties.asMap();
    fillPlaceholderValue(desiredMap, Integer.toString(brokerId));
    JsonNode source = patchMapper().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true).valueToTree(currentMap);
    JsonNode target = patchMapper().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true).valueToTree(desiredMap);
    JsonNode jsonDiff = JsonDiff.asJson(source, target);
    for (JsonNode d : jsonDiff) {
        String pathValue = d.get("path").asText();
        String pathValueWithoutSlash = pathValue.substring(1);
        Optional<ConfigEntry> optEntry = brokerConfigs.entries().stream().filter(configEntry -> configEntry.name().equals(pathValueWithoutSlash)).findFirst();
        String op = d.get("op").asText();
        if (optEntry.isPresent()) {
            ConfigEntry entry = optEntry.get();
            if ("remove".equals(op)) {
                removeProperty(configModel, updatedCE, pathValueWithoutSlash, entry);
            } else if ("replace".equals(op)) {
                // entry is in the current, desired is updated value
                updateOrAdd(entry.name(), configModel, desiredMap, updatedCE);
            }
        } else {
            if ("add".equals(op)) {
                // entry is not in the current, it is added
                updateOrAdd(pathValueWithoutSlash, configModel, desiredMap, updatedCE);
            }
        }
        if ("remove".equals(op)) {
            // there is a lot of properties set by default - not having them in desired causes very noisy log output
            LOGGER.traceCr(reconciliation, "Kafka Broker {} Config Differs : {}", brokerId, d);
            LOGGER.traceCr(reconciliation, "Current Kafka Broker Config path {} has value {}", pathValueWithoutSlash, lookupPath(source, pathValue));
            LOGGER.traceCr(reconciliation, "Desired Kafka Broker Config path {} has value {}", pathValueWithoutSlash, lookupPath(target, pathValue));
        } else {
            LOGGER.debugCr(reconciliation, "Kafka Broker {} Config Differs : {}", brokerId, d);
            LOGGER.debugCr(reconciliation, "Current Kafka Broker Config path {} has value {}", pathValueWithoutSlash, lookupPath(source, pathValue));
            LOGGER.debugCr(reconciliation, "Desired Kafka Broker Config path {} has value {}", pathValueWithoutSlash, lookupPath(target, pathValue));
        }
    }
    return updatedCE;
}

Example 8 with KafkaVersion

use of io.strimzi.operator.cluster.model.KafkaVersion in project strimzi by strimzi.

the class KafkaRoller method checkReconfigurability.

/**
 * Determine whether the pod should be restarted, or the broker reconfigured.
 */
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private void checkReconfigurability(PodRef podRef, Pod pod, RestartContext restartContext) throws ForceableProblem, InterruptedException, FatalProblem {
    List<String> reasonToRestartPod = Objects.requireNonNull(podNeedsRestart.apply(pod));
    boolean podStuck = pod != null && pod.getStatus() != null && "Pending".equals(pod.getStatus().getPhase()) && pod.getStatus().getConditions().stream().anyMatch(ps -> "PodScheduled".equals(ps.getType()) && "Unschedulable".equals(ps.getReason()) && "False".equals(ps.getStatus()));
    if (podStuck && // "Pod has old generation" is used with StatefulSets
    !reasonToRestartPod.contains("Pod has old generation") && !reasonToRestartPod.contains("Pod has old revision")) {
        // and deleting a different pod in the meantime will likely result in another unschedulable pod.
        throw new FatalProblem("Pod is unschedulable");
    }
    // Unless the annotation is present, check the pod is at least ready.
    boolean needsRestart = !reasonToRestartPod.isEmpty();
    KafkaBrokerConfigurationDiff diff = null;
    KafkaBrokerLoggingConfigurationDiff loggingDiff = null;
    boolean needsReconfig = false;
    // connect to the broker and that it's capable of responding.
    if (!initAdminClient()) {
        LOGGER.infoCr(reconciliation, "Pod {} needs to be restarted, because it does not seem to responding to connection attempts", podRef);
        restartContext.needsRestart = false;
        restartContext.needsReconfig = false;
        restartContext.forceRestart = true;
        restartContext.diff = null;
        restartContext.logDiff = null;
        return;
    }
    Config brokerConfig;
    try {
        brokerConfig = brokerConfig(podRef);
    } catch (ForceableProblem e) {
        if (restartContext.backOff.done()) {
            needsRestart = true;
            brokerConfig = null;
        } else {
            throw e;
        }
    }
    if (!needsRestart && allowReconfiguration) {
        LOGGER.traceCr(reconciliation, "Broker {}: description {}", podRef, brokerConfig);
        diff = new KafkaBrokerConfigurationDiff(reconciliation, brokerConfig, kafkaConfigProvider.apply(podRef.getPodId()), kafkaVersion, podRef.getPodId());
        loggingDiff = logging(podRef);
        if (diff.getDiffSize() > 0) {
            if (diff.canBeUpdatedDynamically()) {
                LOGGER.debugCr(reconciliation, "Pod {} needs to be reconfigured.", podRef);
                needsReconfig = true;
            } else {
                LOGGER.infoCr(reconciliation, "Pod {} needs to be restarted, because reconfiguration cannot be done dynamically", podRef);
                needsRestart = true;
            }
        }
        // needsRestart value might have changed from the check in the parent if. So we need to check it again.
        if (!needsRestart && loggingDiff.getDiffSize() > 0) {
            LOGGER.debugCr(reconciliation, "Pod {} logging needs to be reconfigured.", podRef);
            needsReconfig = true;
        }
    } else if (needsRestart) {
        LOGGER.infoCr(reconciliation, "Pod {} needs to be restarted. Reason: {}", podRef, reasonToRestartPod);
    }
    if (podStuck) {
        LOGGER.infoCr(reconciliation, "Pod {} needs to be restarted, because it seems to be stuck and restart might help", podRef);
    }
    restartContext.needsRestart = needsRestart;
    restartContext.needsReconfig = needsReconfig;
    restartContext.forceRestart = podStuck;
    restartContext.diff = diff;
    restartContext.logDiff = loggingDiff;
}

Example 9 with KafkaVersion

use of io.strimzi.operator.cluster.model.KafkaVersion in project strimzi by strimzi.

the class KafkaUpgradeDowngradeMockTest method testUpgradeFromUnsupportedKafkaVersionWithMessageAndProtocol.

// Tests upgrade from Kafka version not supported by the current version of the operator with message format and
// protocol versions specified.
@Test
public void testUpgradeFromUnsupportedKafkaVersionWithMessageAndProtocol(VertxTestContext context) {
    KafkaVersion unsupported = VERSIONS.version("2.1.0");
    Kafka initialKafka = kafkaWithVersions(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion());
    Kafka updatedKafka = kafkaWithVersions(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion());
    Checkpoint reconciliation = context.checkpoint();
    initialize(context, initialKafka).onComplete(context.succeeding(v -> {
        context.verify(() -> {
            assertVersionsInStatefulSet(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion(), KafkaVersionTestUtils.LATEST_KAFKA_IMAGE);
        });
    })).compose(v -> {
        StatefulSet sts = client.apps().statefulSets().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka").get();
        StatefulSet modifiedSts = new StatefulSetBuilder(sts).editMetadata().addToAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION, unsupported.version()).endMetadata().editSpec().editTemplate().editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_LOG_MESSAGE_FORMAT_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_INTER_BROKER_PROTOCOL_VERSION).endMetadata().editSpec().editContainer(0).withImage("strimzi/kafka:old-kafka-2.1.0").endContainer().endSpec().endTemplate().endSpec().build();
        client.apps().statefulSets().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka").createOrReplace(modifiedSts);
        for (int i = 0; i < 3; i++) {
            Pod pod = client.pods().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka-" + i).get();
            Pod modifiedPod = new PodBuilder(pod).editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_LOG_MESSAGE_FORMAT_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_INTER_BROKER_PROTOCOL_VERSION).endMetadata().editSpec().editContainer(0).withImage("strimzi/kafka:old-kafka-2.1.0").endContainer().endSpec().build();
            client.pods().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka-" + i).createOrReplace(modifiedPod);
        }
        return Future.succeededFuture();
    }).compose(v -> operator.createOrUpdate(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME), updatedKafka)).onComplete(context.succeeding(v -> context.verify(() -> {
        assertVersionsInStatefulSet(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion(), KafkaVersionTestUtils.LATEST_KAFKA_IMAGE);
        reconciliation.flag();
    })));
}

Example 10 with KafkaVersion

use of io.strimzi.operator.cluster.model.KafkaVersion in project strimzi by strimzi.

the class KafkaUpgradeDowngradeMockTest method testUpgradeWithoutAnyVersionInPodsOrStsFails.

// Tests upgrade when Kafka StatefulSet and/or Pods existing but without any of the version annotations. This
// indicates that the Statefulset / Pods were not using the current or recent Strimzi version and since we do not
// know the version, we should wail.
@Test
public void testUpgradeWithoutAnyVersionInPodsOrStsFails(VertxTestContext context) {
    KafkaVersion unsupported = VERSIONS.version("2.1.0");
    Kafka initialKafka = kafkaWithVersions(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion());
    Kafka updatedKafka = kafkaWithVersions(KafkaVersionTestUtils.LATEST_KAFKA_VERSION);
    Checkpoint reconciliation = context.checkpoint();
    initialize(context, initialKafka).onComplete(context.succeeding(v -> {
        context.verify(() -> {
            assertVersionsInStatefulSet(KafkaVersionTestUtils.LATEST_KAFKA_VERSION, unsupported.messageVersion(), unsupported.protocolVersion(), KafkaVersionTestUtils.LATEST_KAFKA_IMAGE);
        });
    })).compose(v -> {
        StatefulSet sts = client.apps().statefulSets().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka").get();
        StatefulSet modifiedSts = new StatefulSetBuilder(sts).editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).endMetadata().editSpec().editTemplate().editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_LOG_MESSAGE_FORMAT_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_INTER_BROKER_PROTOCOL_VERSION).endMetadata().endTemplate().endSpec().build();
        client.apps().statefulSets().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka").createOrReplace(modifiedSts);
        for (int i = 0; i < 3; i++) {
            Pod pod = client.pods().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka-" + i).get();
            Pod modifiedPod = new PodBuilder(pod).editMetadata().removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_KAFKA_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_LOG_MESSAGE_FORMAT_VERSION).removeFromAnnotations(KafkaCluster.ANNO_STRIMZI_IO_INTER_BROKER_PROTOCOL_VERSION).endMetadata().build();
            client.pods().inNamespace(NAMESPACE).withName(CLUSTER_NAME + "-kafka-" + i).createOrReplace(modifiedPod);
        }
        return Future.succeededFuture();
    }).compose(v -> operator.createOrUpdate(new Reconciliation("test-trigger", Kafka.RESOURCE_KIND, NAMESPACE, CLUSTER_NAME), updatedKafka)).onComplete(context.failing(v -> context.verify(() -> {
        assertThat(v.getMessage(), stringContainsInOrder("Kafka Pods or StatefulSet exist, but do not contain the strimzi.io/kafka-version annotation to detect their version. Kafka upgrade cannot be detected."));
        reconciliation.flag();
    })));
}