Examples with InlineLogging io.strimzi.api.kafka.model.InlineLogging used on opensource projects

Example 16 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi by strimzi.

the class LoggingChangeST method testDynamicallySetConnectLoggingLevels.

@ParallelNamespaceTest
@Tag(ROLLING_UPDATE)
@Tag(CONNECT)
@Tag(CONNECT_COMPONENTS)
void testDynamicallySetConnectLoggingLevels(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final String kafkaClientsName = mapWithKafkaClientNames.get(extensionContext.getDisplayName());
    InlineLogging ilOff = new InlineLogging();
    Map<String, String> loggers = new HashMap<>();
    loggers.put("connect.root.logger.level", "OFF");
    ilOff.setLoggers(loggers);
    // create async
    resourceManager.createResource(extensionContext, false, KafkaTemplates.kafkaEphemeral(clusterName, 3).build());
    resourceManager.createResource(extensionContext, false, KafkaClientsTemplates.kafkaClients(false, kafkaClientsName).build());
    // sync point
    KafkaUtils.waitForKafkaReady(namespaceName, clusterName);
    final String kafkaClientsPodName = PodUtils.getPodsByPrefixInNameWithDynamicWait(namespaceName, kafkaClientsName).get(0).getMetadata().getName();
    resourceManager.createResource(extensionContext, false, KafkaConnectTemplates.kafkaConnect(extensionContext, clusterName, 1).editSpec().withInlineLogging(ilOff).endSpec().editMetadata().addToAnnotations(Annotations.STRIMZI_IO_USE_CONNECTOR_RESOURCES, "true").endMetadata().build());
    KafkaConnectUtils.waitForConnectReady(namespaceName, clusterName);
    Map<String, String> connectSnapshot = DeploymentUtils.depSnapshot(namespaceName, KafkaConnectResources.deploymentName(clusterName));
    final String connectPodName = connectSnapshot.keySet().iterator().next();
    LOGGER.info("Asserting if log is without records");
    assertFalse(DEFAULT_LOG4J_PATTERN.matcher(StUtils.getLogFromPodByTime(namespaceName, connectPodName, "", "30s")).find());
    LOGGER.info("Changing rootLogger level to DEBUG with inline logging");
    InlineLogging ilDebug = new InlineLogging();
    loggers.put("connect.root.logger.level", "DEBUG");
    ilDebug.setLoggers(loggers);
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, conn -> {
        conn.getSpec().setLogging(ilDebug);
    }, namespaceName);
    LOGGER.info("Waiting for log4j.properties will contain desired settings");
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(namespaceName).execInPod(kafkaClientsPodName, "curl", "http://" + KafkaConnectResources.serviceName(clusterName) + ":8083/admin/loggers/root").out().contains("DEBUG"));
    TestUtils.waitFor("log to not be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        String kcLog = StUtils.getLogFromPodByTime(namespaceName, connectPodName, "", "30s");
        return kcLog != null && !kcLog.isEmpty() && DEFAULT_LOG4J_PATTERN.matcher(kcLog).find();
    });
    String log4jConfig = "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\n" + "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\n" + "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %p %X{connector.context}%m (%c) [%t]%n\n" + "log4j.rootLogger=OFF, CONSOLE\n" + "log4j.logger.org.apache.zookeeper=ERROR\n" + "log4j.logger.org.I0Itec.zkclient=ERROR\n" + "log4j.logger.org.reflections=ERROR";
    String externalCmName = "external-cm";
    ConfigMap connectLoggingMap = new ConfigMapBuilder().withNewMetadata().addToLabels("app", "strimzi").withName(externalCmName).withNamespace(namespaceName).endMetadata().withData(Collections.singletonMap("log4j.properties", log4jConfig)).build();
    kubeClient().getClient().configMaps().inNamespace(namespaceName).createOrReplace(connectLoggingMap);
    ExternalLogging connectXternalLogging = new ExternalLoggingBuilder().withNewValueFrom().withConfigMapKeyRef(new ConfigMapKeySelectorBuilder().withName(externalCmName).withKey("log4j.properties").build()).endValueFrom().build();
    LOGGER.info("Setting log level of Connect to OFF");
    // change to the external logging
    KafkaConnectResource.replaceKafkaConnectResourceInSpecificNamespace(clusterName, conn -> {
        conn.getSpec().setLogging(connectXternalLogging);
    }, namespaceName);
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(namespaceName).execInPod(kafkaClientsPodName, "curl", "http://" + KafkaConnectResources.serviceName(clusterName) + ":8083/admin/loggers/root").out().contains("OFF"));
    TestUtils.waitFor("log to be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        String kcLog = StUtils.getLogFromPodByTime(namespaceName, connectPodName, "", "30s");
        return kcLog != null && kcLog.isEmpty() && !DEFAULT_LOG4J_PATTERN.matcher(kcLog).find();
    });
    assertThat("Connect pod should not roll", DeploymentUtils.depSnapshot(namespaceName, KafkaConnectResources.deploymentName(clusterName)), equalTo(connectSnapshot));
}

Example 17 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi by strimzi.

the class LoggingChangeST method testDynamicallySetUnknownKafkaLogger.

@ParallelNamespaceTest
void testDynamicallySetUnknownKafkaLogger(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 1).build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    InlineLogging il = new InlineLogging();
    il.setLoggers(Collections.singletonMap("log4j.logger.paprika", "INFO"));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setLogging(il);
    }, namespaceName);
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, kafkaPods);
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(namespaceName).execInPodContainer(Level.TRACE, KafkaResources.kafkaPodName(clusterName, 0), "kafka", "/bin/bash", "-c", "bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type broker-loggers --entity-name 0").out().contains("paprika=INFO"));
}

Example 18 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi by strimzi.

the class LoggingChangeST method testDynamicallySetUnknownKafkaLoggerValue.

@ParallelNamespaceTest
void testDynamicallySetUnknownKafkaLoggerValue(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 1).build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    InlineLogging il = new InlineLogging();
    il.setLoggers(Collections.singletonMap("kafka.root.logger.level", "PAPRIKA"));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setLogging(il);
    }, namespaceName);
    RollingUpdateUtils.waitForNoRollingUpdate(namespaceName, kafkaSelector, kafkaPods);
    assertThat("Kafka pod should not roll", RollingUpdateUtils.componentHasRolled(namespaceName, kafkaSelector, kafkaPods), is(false));
}

Example 19 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi by strimzi.

the class ZookeeperCluster method fromCrd.

@SuppressWarnings({ "checkstyle:MethodLength", "checkstyle:CyclomaticComplexity" })
public static ZookeeperCluster fromCrd(Reconciliation reconciliation, Kafka kafkaAssembly, KafkaVersion.Lookup versions, Storage oldStorage, int oldReplicas) {
    ZookeeperCluster zk = new ZookeeperCluster(reconciliation, kafkaAssembly);
    zk.setOwnerReference(kafkaAssembly);
    ZookeeperClusterSpec zookeeperClusterSpec = kafkaAssembly.getSpec().getZookeeper();
    int replicas = zookeeperClusterSpec.getReplicas();
    if (replicas <= 0) {
        replicas = ZookeeperClusterSpec.DEFAULT_REPLICAS;
    }
    if (replicas == 1 && zookeeperClusterSpec.getStorage() != null && "ephemeral".equals(zookeeperClusterSpec.getStorage().getType())) {
        LOGGER.warnCr(reconciliation, "A ZooKeeper cluster with a single replica and ephemeral storage will be in a defective state after any restart or rolling update. It is recommended that a minimum of three replicas are used.");
    }
    zk.setReplicas(replicas);
    // Get the ZK version information from either the CRD or from the default setting
    KafkaClusterSpec kafkaClusterSpec = kafkaAssembly.getSpec().getKafka();
    String version = versions.supportedVersion(kafkaClusterSpec != null ? kafkaClusterSpec.getVersion() : null).zookeeperVersion();
    zk.setVersion(version);
    String image = zookeeperClusterSpec.getImage();
    if (image == null) {
        image = versions.kafkaImage(kafkaClusterSpec != null ? kafkaClusterSpec.getImage() : null, kafkaClusterSpec != null ? kafkaClusterSpec.getVersion() : null);
    }
    zk.setImage(image);
    if (zookeeperClusterSpec.getReadinessProbe() != null) {
        zk.setReadinessProbe(zookeeperClusterSpec.getReadinessProbe());
    }
    if (zookeeperClusterSpec.getLivenessProbe() != null) {
        zk.setLivenessProbe(zookeeperClusterSpec.getLivenessProbe());
    }
    Logging logging = zookeeperClusterSpec.getLogging();
    zk.setLogging(logging == null ? new InlineLogging() : logging);
    zk.setGcLoggingEnabled(zookeeperClusterSpec.getJvmOptions() == null ? DEFAULT_JVM_GC_LOGGING_ENABLED : zookeeperClusterSpec.getJvmOptions().isGcLoggingEnabled());
    if (zookeeperClusterSpec.getJvmOptions() != null) {
        zk.setJavaSystemProperties(zookeeperClusterSpec.getJvmOptions().getJavaSystemProperties());
    }
    // Parse different types of metrics configurations
    ModelUtils.parseMetrics(zk, zookeeperClusterSpec);
    if (oldStorage != null) {
        Storage newStorage = zookeeperClusterSpec.getStorage();
        AbstractModel.validatePersistentStorage(newStorage);
        StorageDiff diff = new StorageDiff(reconciliation, oldStorage, newStorage, oldReplicas, zookeeperClusterSpec.getReplicas());
        if (!diff.isEmpty()) {
            LOGGER.warnCr(reconciliation, "Only the following changes to Zookeeper storage are allowed: " + "changing the deleteClaim flag, " + "changing overrides to nodes which do not exist yet " + "and increasing size of persistent claim volumes (depending on the volume type and used storage class).");
            LOGGER.warnCr(reconciliation, "The desired ZooKeeper storage configuration in the custom resource {}/{} contains changes which are not allowed. As " + "a result, all storage changes will be ignored. Use DEBUG level logging for more information " + "about the detected changes.", kafkaAssembly.getMetadata().getNamespace(), kafkaAssembly.getMetadata().getName());
            Condition warning = StatusUtils.buildWarningCondition("ZooKeeperStorage", "The desired ZooKeeper storage configuration contains changes which are not allowed. As a " + "result, all storage changes will be ignored. Use DEBUG level logging for more information " + "about the detected changes.");
            zk.addWarningCondition(warning);
            zk.setStorage(oldStorage);
        } else {
            zk.setStorage(newStorage);
        }
    } else {
        zk.setStorage(zookeeperClusterSpec.getStorage());
    }
    zk.setConfiguration(new ZookeeperConfiguration(reconciliation, zookeeperClusterSpec.getConfig().entrySet()));
    zk.setResources(zookeeperClusterSpec.getResources());
    zk.setJvmOptions(zookeeperClusterSpec.getJvmOptions());
    if (zookeeperClusterSpec.getJmxOptions() != null) {
        zk.setJmxEnabled(Boolean.TRUE);
        AuthenticationUtils.configureZookeeperJmxOptions(zookeeperClusterSpec.getJmxOptions().getAuthentication(), zk);
    }
    if (zookeeperClusterSpec.getTemplate() != null) {
        ZookeeperClusterTemplate template = zookeeperClusterSpec.getTemplate();
        if (template.getStatefulset() != null) {
            if (template.getStatefulset().getPodManagementPolicy() != null) {
                zk.templatePodManagementPolicy = template.getStatefulset().getPodManagementPolicy();
            }
            if (template.getStatefulset().getMetadata() != null) {
                zk.templateStatefulSetLabels = template.getStatefulset().getMetadata().getLabels();
                zk.templateStatefulSetAnnotations = template.getStatefulset().getMetadata().getAnnotations();
            }
        }
        if (template.getPodSet() != null && template.getPodSet().getMetadata() != null) {
            zk.templatePodSetLabels = template.getPodSet().getMetadata().getLabels();
            zk.templatePodSetAnnotations = template.getPodSet().getMetadata().getAnnotations();
        }
        ModelUtils.parsePodTemplate(zk, template.getPod());
        ModelUtils.parseInternalServiceTemplate(zk, template.getClientService());
        ModelUtils.parseInternalHeadlessServiceTemplate(zk, template.getNodesService());
        if (template.getPersistentVolumeClaim() != null && template.getPersistentVolumeClaim().getMetadata() != null) {
            zk.templatePersistentVolumeClaimLabels = Util.mergeLabelsOrAnnotations(template.getPersistentVolumeClaim().getMetadata().getLabels(), zk.templateStatefulSetLabels);
            zk.templatePersistentVolumeClaimAnnotations = template.getPersistentVolumeClaim().getMetadata().getAnnotations();
        }
        if (template.getZookeeperContainer() != null && template.getZookeeperContainer().getEnv() != null) {
            zk.templateZookeeperContainerEnvVars = template.getZookeeperContainer().getEnv();
        }
        if (template.getZookeeperContainer() != null && template.getZookeeperContainer().getSecurityContext() != null) {
            zk.templateZookeeperContainerSecurityContext = template.getZookeeperContainer().getSecurityContext();
        }
        if (template.getServiceAccount() != null && template.getServiceAccount().getMetadata() != null) {
            zk.templateServiceAccountLabels = template.getServiceAccount().getMetadata().getLabels();
            zk.templateServiceAccountAnnotations = template.getServiceAccount().getMetadata().getAnnotations();
        }
        if (template.getJmxSecret() != null && template.getJmxSecret().getMetadata() != null) {
            zk.templateJmxSecretLabels = template.getJmxSecret().getMetadata().getLabels();
            zk.templateJmxSecretAnnotations = template.getJmxSecret().getMetadata().getAnnotations();
        }
        ModelUtils.parsePodDisruptionBudgetTemplate(zk, template.getPodDisruptionBudget());
    }
    zk.templatePodLabels = Util.mergeLabelsOrAnnotations(zk.templatePodLabels, DEFAULT_POD_LABELS);
    return zk;
}

Example 20 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi by strimzi.

the class LogSettingST method testCruiseControlLogChange.

@IsolatedTest("Updating shared Kafka")
// This test might be flaky, as it gets real logs from CruiseControl pod
void testCruiseControlLogChange(ExtensionContext extensionContext) {
    final String debugText = " DEBUG ";
    String cruiseControlPodName = PodUtils.getPodNameByPrefix(namespace, LOG_SETTING_CLUSTER_NAME + "-" + Constants.CRUISE_CONTROL_CONTAINER_NAME);
    LOGGER.info("Check that default/actual root logging level is info");
    String containerLogLevel = cmdKubeClient().namespace(namespace).execInPod(cruiseControlPodName, "grep", "-i", "rootlogger.level", Constants.CRUISE_CONTROL_LOG_FILE_PATH).out().trim().split("=")[1];
    assertThat(containerLogLevel.toUpperCase(Locale.ENGLISH), is(not(debugText.strip())));
    LOGGER.info("Check logs in CruiseControl - make sure no DEBUG is found there.");
    String logOut = StUtils.getLogFromPodByTime(namespace, cruiseControlPodName, Constants.CRUISE_CONTROL_CONTAINER_NAME, "20s");
    assertThat(logOut.toUpperCase(Locale.ENGLISH), not(containsString(debugText)));
    InlineLogging logging = new InlineLogging();
    logging.setLoggers(Collections.singletonMap("rootLogger.level", debugText.strip()));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(LOG_SETTING_CLUSTER_NAME, kafka -> kafka.getSpec().getCruiseControl().setLogging(logging), namespace);
    LOGGER.info("Wait for change of root logger in {}.", cruiseControlPodName);
    TestUtils.waitFor("Waiting for log to be changed", CC_LOG_CONFIG_RELOAD, CO_OPERATION_TIMEOUT_MEDIUM, () -> {
        String line = StUtils.getLineFromPodContainer(namespace, cruiseControlPodName, null, Constants.CRUISE_CONTROL_LOG_FILE_PATH, "rootlogger.level");
        return line.toUpperCase(Locale.ENGLISH).contains(debugText.strip());
    });
    LOGGER.info("Check cruise control logs in pod {} and it's container {} .", cruiseControlPodName, Constants.CRUISE_CONTROL_CONTAINER_NAME);
    TestUtils.waitFor("Wait for debug log line to show in logs", CC_LOG_CONFIG_RELOAD, TIMEOUT_FOR_LOG, () -> {
        String log = StUtils.getLogFromPodByTime(namespace, cruiseControlPodName, Constants.CRUISE_CONTROL_CONTAINER_NAME, "20s");
        return log.toUpperCase(Locale.ENGLISH).contains(debugText);
    });
}