Examples with InlineLogging io.strimzi.api.kafka.model.InlineLogging used on opensource projects

Example 31 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi-kafka-operator by strimzi.

the class LoggingChangeST method testDynamicallySetBridgeLoggingLevels.

@ParallelNamespaceTest
@Tag(BRIDGE)
@Tag(ROLLING_UPDATE)
void testDynamicallySetBridgeLoggingLevels(ExtensionContext extensionContext) {
    final TestStorage testStorage = new TestStorage(extensionContext);
    InlineLogging ilOff = new InlineLogging();
    Map<String, String> loggers = new HashMap<>();
    loggers.put("rootLogger.level", "OFF");
    loggers.put("logger.bridge.level", "OFF");
    loggers.put("logger.healthy.level", "OFF");
    loggers.put("logger.ready.level", "OFF");
    ilOff.setLoggers(loggers);
    // create resources async
    resourceManager.createResource(extensionContext, false, KafkaTemplates.kafkaPersistent(testStorage.getClusterName(), 1, 1).build(), ScraperTemplates.scraperPod(testStorage.getNamespaceName(), testStorage.getScraperName()).build(), KafkaBridgeTemplates.kafkaBridge(testStorage.getClusterName(), KafkaResources.tlsBootstrapAddress(testStorage.getClusterName()), 1).editSpec().withInlineLogging(ilOff).endSpec().build());
    KafkaUtils.waitForKafkaReady(testStorage.getNamespaceName(), testStorage.getClusterName());
    DeploymentUtils.waitForDeploymentReady(testStorage.getNamespaceName(), testStorage.getScraperName());
    KafkaBridgeUtils.waitForKafkaBridgeReady(testStorage.getNamespaceName(), testStorage.getClusterName());
    Map<String, String> bridgeSnapshot = DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaBridgeResources.deploymentName(testStorage.getClusterName()));
    final String bridgePodName = bridgeSnapshot.keySet().iterator().next();
    LOGGER.info("Asserting if log is without records");
    assertFalse(DEFAULT_LOG4J_PATTERN.matcher(StUtils.getLogFromPodByTime(testStorage.getNamespaceName(), bridgePodName, "", "30s")).find());
    LOGGER.info("Changing rootLogger level to DEBUG with inline logging");
    InlineLogging ilDebug = new InlineLogging();
    loggers.put("rootLogger.level", "DEBUG");
    loggers.put("logger.bridge.level", "OFF");
    loggers.put("logger.healthy.level", "OFF");
    loggers.put("logger.ready.level", "OFF");
    ilDebug.setLoggers(loggers);
    KafkaBridgeResource.replaceBridgeResourceInSpecificNamespace(testStorage.getClusterName(), bridz -> {
        bridz.getSpec().setLogging(ilDebug);
    }, testStorage.getNamespaceName());
    LOGGER.info("Waiting for log4j2.properties will contain desired settings");
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(testStorage.getNamespaceName()).execInPodContainer(Level.TRACE, bridgePodName, KafkaBridgeResources.deploymentName(testStorage.getClusterName()), "cat", "/opt/strimzi/custom-config/log4j2.properties").out().contains("rootLogger.level=DEBUG") && cmdKubeClient().namespace(testStorage.getNamespaceName()).execInPodContainer(Level.TRACE, bridgePodName, KafkaBridgeResources.deploymentName(testStorage.getClusterName()), "cat", "/opt/strimzi/custom-config/log4j2.properties").out().contains("monitorInterval=30"));
    TestUtils.waitFor("log to not be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        String bridgeLog = StUtils.getLogFromPodByTime(testStorage.getNamespaceName(), bridgePodName, KafkaBridgeResources.deploymentName(testStorage.getClusterName()), "30s");
        return bridgeLog != null && !bridgeLog.isEmpty() && DEFAULT_LOG4J_PATTERN.matcher(bridgeLog).find();
    });
    ConfigMap configMapBridge = new ConfigMapBuilder().withNewMetadata().withName("external-configmap-bridge").withNamespace(testStorage.getNamespaceName()).endMetadata().withData(Collections.singletonMap("log4j2.properties", "name = BridgeConfig\n" + "\n" + "appender.console.type = Console\n" + "appender.console.name = STDOUT\n" + "appender.console.layout.type = PatternLayout\n" + "appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n\n" + "\n" + "rootLogger.level = OFF\n" + "rootLogger.appenderRefs = console\n" + "rootLogger.appenderRef.console.ref = STDOUT\n" + "rootLogger.additivity = false\n" + "\n" + "logger.bridge.name = io.strimzi.kafka.bridge\n" + "logger.bridge.level = OFF\n" + "logger.bridge.appenderRefs = console\n" + "logger.bridge.appenderRef.console.ref = STDOUT\n" + "logger.bridge.additivity = false\n" + "\n" + "# HTTP OpenAPI specific logging levels (default is INFO)\n" + "# Logging healthy and ready endpoints is very verbose because of Kubernetes health checking.\n" + "logger.healthy.name = http.openapi.operation.healthy\n" + "logger.healthy.level = OFF\n" + "logger.ready.name = http.openapi.operation.ready\n" + "logger.ready.level = OFF")).build();
    kubeClient().getClient().configMaps().inNamespace(testStorage.getNamespaceName()).createOrReplace(configMapBridge);
    ExternalLogging bridgeXternalLogging = new ExternalLoggingBuilder().withNewValueFrom().withConfigMapKeyRef(new ConfigMapKeySelectorBuilder().withName("external-configmap-bridge").withKey("log4j2.properties").build()).endValueFrom().build();
    LOGGER.info("Setting log level of Bridge to OFF - records should not appear in the log");
    // change to the external logging
    KafkaBridgeResource.replaceBridgeResourceInSpecificNamespace(testStorage.getClusterName(), bridz -> {
        bridz.getSpec().setLogging(bridgeXternalLogging);
    }, testStorage.getNamespaceName());
    LOGGER.info("Waiting for log4j2.properties will contain desired settings");
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(testStorage.getNamespaceName()).execInPodContainer(Level.TRACE, bridgePodName, KafkaBridgeResources.deploymentName(testStorage.getClusterName()), "cat", "/opt/strimzi/custom-config/log4j2.properties").out().contains("rootLogger.level = OFF") && cmdKubeClient().namespace(testStorage.getNamespaceName()).execInPodContainer(Level.TRACE, bridgePodName, KafkaBridgeResources.deploymentName(testStorage.getClusterName()), "cat", "/opt/strimzi/custom-config/log4j2.properties").out().contains("monitorInterval=30"));
    TestUtils.waitFor("log to be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        String bridgeLog = StUtils.getLogFromPodByTime(testStorage.getNamespaceName(), bridgePodName, KafkaBridgeResources.deploymentName(testStorage.getClusterName()), "30s");
        return bridgeLog != null && bridgeLog.isEmpty() && !DEFAULT_LOG4J_PATTERN.matcher(bridgeLog).find();
    });
    assertThat("Bridge pod should not roll", DeploymentUtils.depSnapshot(testStorage.getNamespaceName(), KafkaBridgeResources.deploymentName(testStorage.getClusterName())), equalTo(bridgeSnapshot));
}

Example 32 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi-kafka-operator by strimzi.

the class LoggingChangeST method testDynamicallySetKafkaLoggingLevels.

@ParallelNamespaceTest
void testDynamicallySetKafkaLoggingLevels(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String kafkaName = KafkaResources.kafkaStatefulSetName(clusterName);
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, kafkaName);
    InlineLogging ilOff = new InlineLogging();
    Map<String, String> log4jConfig = new HashMap<>();
    log4jConfig.put("kafka.root.logger.level", "OFF");
    log4jConfig.put("log4j.logger.org.I0Itec.zkclient.ZkClient", "OFF");
    log4jConfig.put("log4j.logger.org.apache.zookeeper", "OFF");
    log4jConfig.put("log4j.logger.kafka", "OFF");
    log4jConfig.put("log4j.logger.org.apache.kafka", "OFF");
    log4jConfig.put("log4j.logger.kafka.request.logger", "OFF, CONSOLE");
    log4jConfig.put("log4j.logger.kafka.network.Processor", "OFF");
    log4jConfig.put("log4j.logger.kafka.server.KafkaApis", "OFF");
    log4jConfig.put("log4j.logger.kafka.network.RequestChannel$", "OFF");
    log4jConfig.put("log4j.logger.kafka.controller", "OFF");
    log4jConfig.put("log4j.logger.kafka.log.LogCleaner", "OFF");
    log4jConfig.put("log4j.logger.state.change.logger", "OFF");
    log4jConfig.put("log4j.logger.kafka.authorizer.logger", "OFF");
    ilOff.setLoggers(log4jConfig);
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaEphemeral(clusterName, 3, 1).editSpec().editKafka().withInlineLogging(ilOff).endKafka().endSpec().build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    TestUtils.waitFor("log to not be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        boolean[] correctLogging = { true };
        kafkaPods.keySet().forEach(podName -> {
            String kafkaLog = StUtils.getLogFromPodByTime(namespaceName, podName, "kafka", "30s");
            correctLogging[0] = kafkaLog != null && kafkaLog.isEmpty() && !DEFAULT_LOG4J_PATTERN.matcher(kafkaLog).find();
        });
        return correctLogging[0];
    });
    LOGGER.info("Changing rootLogger level to DEBUG with inline logging");
    InlineLogging ilDebug = new InlineLogging();
    ilDebug.setLoggers(Collections.singletonMap("kafka.root.logger.level", "DEBUG"));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setLogging(ilDebug);
    }, namespaceName);
    LOGGER.info("Waiting for dynamic change in the kafka pod");
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(namespaceName).execInPodContainer(Level.TRACE, KafkaResources.kafkaPodName(clusterName, 0), "kafka", "/bin/bash", "-c", "bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type broker-loggers --entity-name 0").out().contains("root=DEBUG"));
    TestUtils.waitFor("log to not be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        boolean[] correctLogging = { true };
        kafkaPods.keySet().forEach(podName -> {
            String kafkaLog = StUtils.getLogFromPodByTime(namespaceName, podName, "kafka", "30s");
            correctLogging[0] = kafkaLog != null && !kafkaLog.isEmpty() && DEFAULT_LOG4J_PATTERN.matcher(kafkaLog).find();
        });
        return correctLogging[0];
    });
    LOGGER.info("Setting external logging INFO");
    ConfigMap configMap = new ConfigMapBuilder().withNewMetadata().withName("external-configmap").withNamespace(namespaceName).endMetadata().withData(Collections.singletonMap("log4j.properties", "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\n" + "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\n" + "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %p %m (%c) [%t]%n\n" + "log4j.rootLogger=INFO, CONSOLE\n" + "log4j.logger.org.I0Itec.zkclient.ZkClient=INFO\n" + "log4j.logger.org.apache.zookeeper=INFO\n" + "log4j.logger.kafka=INFO\n" + "log4j.logger.org.apache.kafka=INFO\n" + "log4j.logger.kafka.request.logger=WARN\n" + "log4j.logger.kafka.network.Processor=ERROR\n" + "log4j.logger.kafka.server.KafkaApis=ERROR\n" + "log4j.logger.kafka.network.RequestChannel$=WARN\n" + "log4j.logger.kafka.controller=TRACE\n" + "log4j.logger.kafka.log.LogCleaner=INFO\n" + "log4j.logger.state.change.logger=TRACE\n" + "log4j.logger.kafka.authorizer.logger=INFO")).build();
    kubeClient().getClient().configMaps().inNamespace(namespaceName).createOrReplace(configMap);
    ExternalLogging elKafka = new ExternalLoggingBuilder().withNewValueFrom().withConfigMapKeyRef(new ConfigMapKeySelectorBuilder().withKey("log4j.properties").withName("external-configmap").withOptional(false).build()).endValueFrom().build();
    LOGGER.info("Setting log level of kafka INFO");
    // change to external logging
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setLogging(elKafka);
    }, namespaceName);
    LOGGER.info("Waiting for dynamic change in the kafka pod");
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(namespaceName).execInPodContainer(Level.TRACE, KafkaResources.kafkaPodName(clusterName, 0), "kafka", "/bin/bash", "-c", "bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type broker-loggers --entity-name 0").out().contains("root=INFO"));
    TestUtils.waitFor("log to not be empty", Duration.ofMillis(100).toMillis(), Constants.SAFETY_RECONCILIATION_INTERVAL, () -> {
        boolean[] correctLogging = { true };
        kafkaPods.keySet().forEach(podName -> {
            String kafkaLog = StUtils.getLogFromPodByTime(namespaceName, podName, "kafka", "30s");
            correctLogging[0] = kafkaLog != null && !kafkaLog.isEmpty() && DEFAULT_LOG4J_PATTERN.matcher(kafkaLog).find();
        });
        return correctLogging[0];
    });
    assertThat("Kafka pod should not roll", RollingUpdateUtils.componentHasRolled(namespaceName, kafkaSelector, kafkaPods), is(false));
}

Example 33 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi-kafka-operator by strimzi.

the class LoggingChangeST method testDynamicallySetUnknownKafkaLogger.

@ParallelNamespaceTest
void testDynamicallySetUnknownKafkaLogger(ExtensionContext extensionContext) {
    final String namespaceName = StUtils.getNamespaceBasedOnRbac(namespace, extensionContext);
    final String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    final LabelSelector kafkaSelector = KafkaResource.getLabelSelector(clusterName, KafkaResources.kafkaStatefulSetName(clusterName));
    resourceManager.createResource(extensionContext, KafkaTemplates.kafkaPersistent(clusterName, 3, 1).build());
    Map<String, String> kafkaPods = PodUtils.podSnapshot(namespaceName, kafkaSelector);
    InlineLogging il = new InlineLogging();
    il.setLoggers(Collections.singletonMap("log4j.logger.paprika", "INFO"));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(clusterName, k -> {
        k.getSpec().getKafka().setLogging(il);
    }, namespaceName);
    RollingUpdateUtils.waitTillComponentHasRolled(namespaceName, kafkaSelector, kafkaPods);
    TestUtils.waitFor("Logger change", Constants.GLOBAL_POLL_INTERVAL, Constants.GLOBAL_TIMEOUT, () -> cmdKubeClient().namespace(namespaceName).execInPodContainer(Level.TRACE, KafkaResources.kafkaPodName(clusterName, 0), "kafka", "/bin/bash", "-c", "bin/kafka-configs.sh --bootstrap-server localhost:9092 --describe --entity-type broker-loggers --entity-name 0").out().contains("paprika=INFO"));
}

Example 34 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi-kafka-operator by strimzi.

the class OauthPlainIsolatedST method testProducerConsumerBridge.

@Description("As a oauth bridge, I should be able to send messages to bridge endpoint.")
@ParallelTest
@Tag(BRIDGE)
void testProducerConsumerBridge(ExtensionContext extensionContext) {
    String clusterName = mapWithClusterNames.get(extensionContext.getDisplayName());
    String producerName = OAUTH_PRODUCER_NAME + "-" + clusterName;
    String consumerName = OAUTH_CONSUMER_NAME + "-" + clusterName;
    String topicName = mapWithTestTopics.get(extensionContext.getDisplayName());
    KafkaOauthClients oauthExampleClients = new KafkaOauthClientsBuilder().withNamespaceName(clusterOperator.getDeploymentNamespace()).withProducerName(producerName).withConsumerName(consumerName).withBootstrapAddress(KafkaResources.plainBootstrapAddress(oauthClusterName)).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).withOauthClientId(OAUTH_CLIENT_NAME).withOauthClientSecret(OAUTH_CLIENT_SECRET).withOauthTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).build();
    resourceManager.createResource(extensionContext, KafkaTopicTemplates.topic(oauthClusterName, topicName, clusterOperator.getDeploymentNamespace()).build());
    resourceManager.createResource(extensionContext, oauthExampleClients.producerStrimziOauthPlain());
    ClientUtils.waitForClientSuccess(producerName, clusterOperator.getDeploymentNamespace(), MESSAGE_COUNT);
    resourceManager.createResource(extensionContext, oauthExampleClients.consumerStrimziOauthPlain());
    ClientUtils.waitForClientSuccess(consumerName, clusterOperator.getDeploymentNamespace(), MESSAGE_COUNT);
    // needed for a verification of oauth configuration
    InlineLogging ilDebug = new InlineLogging();
    ilDebug.setLoggers(Map.of("rootLogger.level", "DEBUG"));
    resourceManager.createResource(extensionContext, KafkaBridgeTemplates.kafkaBridge(oauthClusterName, KafkaResources.plainBootstrapAddress(oauthClusterName), 1).editMetadata().withNamespace(clusterOperator.getDeploymentNamespace()).endMetadata().editSpec().withNewKafkaClientAuthenticationOAuth().withTokenEndpointUri(keycloakInstance.getOauthTokenEndpointUri()).withClientId("kafka-bridge").withNewClientSecret().withSecretName(BRIDGE_OAUTH_SECRET).withKey(OAUTH_KEY).endClientSecret().withConnectTimeoutSeconds(CONNECT_TIMEOUT_S).withReadTimeoutSeconds(READ_TIMEOUT_S).endKafkaClientAuthenticationOAuth().withLogging(ilDebug).endSpec().build());
    final String kafkaBridgePodName = kubeClient().listPods(clusterOperator.getDeploymentNamespace(), oauthClusterName, Labels.STRIMZI_KIND_LABEL, KafkaBridge.RESOURCE_KIND).get(0).getMetadata().getName();
    final String kafkaBridgeLogs = KubeClusterResource.cmdKubeClient(clusterOperator.getDeploymentNamespace()).execInCurrentNamespace(Level.DEBUG, "logs", kafkaBridgePodName).out();
    verifyOauthConfiguration(kafkaBridgeLogs);
    String bridgeProducerName = "bridge-producer-" + clusterName;
    BridgeClients kafkaBridgeClientJob = new BridgeClientsBuilder().withNamespaceName(clusterOperator.getDeploymentNamespace()).withProducerName(bridgeProducerName).withBootstrapAddress(KafkaBridgeResources.serviceName(oauthClusterName)).withTopicName(topicName).withMessageCount(MESSAGE_COUNT).withPort(HTTP_BRIDGE_DEFAULT_PORT).withDelayMs(1000).withPollInterval(1000).build();
    resourceManager.createResource(extensionContext, kafkaBridgeClientJob.producerStrimziBridge());
    ClientUtils.waitForClientSuccess(bridgeProducerName, clusterOperator.getDeploymentNamespace(), MESSAGE_COUNT);
}

Example 35 with InlineLogging

use of io.strimzi.api.kafka.model.InlineLogging in project strimzi-kafka-operator by strimzi.

the class LogSettingST method testCruiseControlLogChange.

@IsolatedTest("Updating shared Kafka")
// This test might be flaky, as it gets real logs from CruiseControl pod
void testCruiseControlLogChange(ExtensionContext extensionContext) {
    final String debugText = " DEBUG ";
    String cruiseControlPodName = PodUtils.getPodNameByPrefix(namespace, LOG_SETTING_CLUSTER_NAME + "-" + Constants.CRUISE_CONTROL_CONTAINER_NAME);
    LOGGER.info("Check that default/actual root logging level is info");
    String containerLogLevel = cmdKubeClient().namespace(namespace).execInPod(cruiseControlPodName, "grep", "-i", "rootlogger.level", Constants.CRUISE_CONTROL_LOG_FILE_PATH).out().trim().split("=")[1];
    assertThat(containerLogLevel.toUpperCase(Locale.ENGLISH), is(not(debugText.strip())));
    LOGGER.info("Check logs in CruiseControl - make sure no DEBUG is found there.");
    String logOut = StUtils.getLogFromPodByTime(namespace, cruiseControlPodName, Constants.CRUISE_CONTROL_CONTAINER_NAME, "20s");
    assertThat(logOut.toUpperCase(Locale.ENGLISH), not(containsString(debugText)));
    InlineLogging logging = new InlineLogging();
    logging.setLoggers(Collections.singletonMap("rootLogger.level", debugText.strip()));
    KafkaResource.replaceKafkaResourceInSpecificNamespace(LOG_SETTING_CLUSTER_NAME, kafka -> kafka.getSpec().getCruiseControl().setLogging(logging), namespace);
    LOGGER.info("Wait for change of root logger in {}.", cruiseControlPodName);
    TestUtils.waitFor("Waiting for log to be changed", CC_LOG_CONFIG_RELOAD, CO_OPERATION_TIMEOUT_MEDIUM, () -> {
        String line = StUtils.getLineFromPodContainer(namespace, cruiseControlPodName, null, Constants.CRUISE_CONTROL_LOG_FILE_PATH, "rootlogger.level");
        return line.toUpperCase(Locale.ENGLISH).contains(debugText.strip());
    });
    LOGGER.info("Check cruise control logs in pod {} and it's container {} .", cruiseControlPodName, Constants.CRUISE_CONTROL_CONTAINER_NAME);
    TestUtils.waitFor("Wait for debug log line to show in logs", CC_LOG_CONFIG_RELOAD, TIMEOUT_FOR_LOG, () -> {
        String log = StUtils.getLogFromPodByTime(namespace, cruiseControlPodName, Constants.CRUISE_CONTROL_CONTAINER_NAME, "20s");
        return log.toUpperCase(Locale.ENGLISH).contains(debugText);
    });
}