Examples with StackGresCluster io.stackgres.common.crd.sgcluster.StackGresCluster used on opensource projects

Example 26 with StackGresCluster

use of io.stackgres.common.crd.sgcluster.StackGresCluster in project stackgres by ongres.

the class PrometheusIntegration method generateResource.

@Override
public Stream<HasMetadata> generateResource(StackGresClusterContext context) {
    final StackGresCluster stackGresCluster = context.getSource();
    final Map<String, String> defaultLabels = labelFactory.clusterLabels(stackGresCluster);
    Map<String, String> crossNamespaceLabels = labelFactory.clusterCrossNamespaceLabels(stackGresCluster);
    Seq<HasMetadata> resources = Seq.of(new ServiceBuilder().withNewMetadata().withNamespace(stackGresCluster.getMetadata().getNamespace()).withName(AbstractEnvoy.serviceName(context)).withLabels(ImmutableMap.<String, String>builder().putAll(crossNamespaceLabels).put("container", AbstractEnvoy.NAME).build()).endMetadata().withSpec(new ServiceSpecBuilder().withSelector(defaultLabels).withPorts(new ServicePortBuilder().withProtocol("TCP").withName(AbstractEnvoy.NAME).withPort(8001).build()).build()).build());
    Optional<Stream<HasMetadata>> serviceMonitors = context.getPrometheus().filter(c -> Optional.ofNullable(c.getCreateServiceMonitor()).orElse(false)).map(c -> getServiceMonitors(context, crossNamespaceLabels, c));
    return serviceMonitors.map(hasMetadataStream -> Stream.concat(resources, hasMetadataStream)).orElse(resources);
}

Example 27 with StackGresCluster

use of io.stackgres.common.crd.sgcluster.StackGresCluster in project stackgres by ongres.

the class FluentBit method buildSource.

protected Optional<HasMetadata> buildSource(StackGresClusterContext context) {
    final StackGresCluster cluster = context.getSource();
    if (cluster.getSpec().getDistributedLogs() == null) {
        return Optional.empty();
    }
    final String namespace = cluster.getMetadata().getNamespace();
    final String fluentdRelativeId = cluster.getSpec().getDistributedLogs().getDistributedLogs();
    final String fluentdNamespace = StackGresUtil.getNamespaceFromRelativeId(fluentdRelativeId, namespace);
    final String fluentdServiceName = FluentdUtil.serviceName(StackGresUtil.getNameFromRelativeId(fluentdRelativeId));
    String parsersConfigFile = "" + "[PARSER]\n" + "    Name        postgreslog_firstline\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2}.\\d*\\s\\S{3})" + ",(?<message>.*)\n" + "\n" + "[PARSER]\n" + "    Name        postgreslog_1\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2}.\\d*\\s\\S{3})" + ",(?<message>\"([^\"]*(?:\"\"[^\"]*)*)\"|)\n" + "\n" + "[PARSER]\n" + "    Name        patronilog_firstline\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2},\\d*{3})" + " (?<error_severity>[^:]+): (?<message>.*)\n" + "\n" + "[PARSER]\n" + "    Name        patronilog_1\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2},\\d*{3})" + " (?<error_severity>[^:]+): (?<message>.*)\n" + "\n" + "[PARSER]\n" + "    Name        kubernetes_tag\n" + "    Format      regex\n" + "    Regex       ^[^.]+\\.[^.]+\\.[^.]+\\." + "(?<namespace_name>[^.]+)\\.(?<pod_name>[^.]+)$\n" + "\n";
    final String clusterNamespace = cluster.getMetadata().getNamespace();
    String fluentBitConfigFile = "" + "[SERVICE]\n" + "    Parsers_File      /etc/fluent-bit/parsers.conf\n" + "\n" + "[INPUT]\n" + "    Name              tail\n" + "    Path              " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/postgres*.csv\n" + "    Tag               " + FluentdUtil.POSTGRES_LOG_TYPE + "\n" + "    DB                " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/postgreslog.db\n" + "    Multiline         On\n" + "    Parser_Firstline  postgreslog_firstline\n" + "    Parser_1          postgreslog_1\n" + "    Buffer_Max_Size   2M\n" + "    Skip_Long_Lines   On\n" + "\n" + "[INPUT]\n" + "    Name              tail\n" + "    Key               message\n" + "    Path              " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/patroni*.log\n" + "    Tag               " + FluentdUtil.PATRONI_LOG_TYPE + "\n" + "    DB                " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/patronilog.db\n" + "    Multiline         On\n" + "    Parser_Firstline  patronilog_firstline\n" + "    Parser_1          patronilog_1\n" + "    Buffer_Max_Size   2M\n" + "    Skip_Long_Lines   On\n" + "\n" + "[FILTER]\n" + "    Name         rewrite_tag\n" + "    Match        " + FluentdUtil.POSTGRES_LOG_TYPE + "\n" + "    Rule         $message ^.*$ " + tagName(cluster, FluentdUtil.POSTGRES_LOG_TYPE) + "." + clusterNamespace + ".${HOSTNAME} false\n" + "    Emitter_Name postgres_re_emitted" + "\n" + "[FILTER]\n" + "    Name         rewrite_tag\n" + "    Match        " + FluentdUtil.PATRONI_LOG_TYPE + "\n" + "    Rule         $message ^.*$ " + tagName(cluster, FluentdUtil.PATRONI_LOG_TYPE) + "." + clusterNamespace + ".${HOSTNAME} false\n" + "    Emitter_Name patroni_re_emitted" + "\n" + "[FILTER]\n" + "    Name                 kubernetes\n" + "    Match                " + tagName(cluster, "*") + "\n" + "    Annotations          Off\n" + "    Kube_Tag_Prefix      ''\n" + "    Regex_Parser         kubernetes_tag\n" + "    Buffer_Size          0\n" + "    Kube_Meta_Cache_TTL  60\n" + "\n" + "[OUTPUT]\n" + "    Name              forward\n" + "    Match             " + tagName(cluster, "*") + "\n" + "    Host              " + fluentdServiceName + "." + fluentdNamespace + "\n" + "    Port              " + FluentdUtil.FORWARD_PORT + "\n" + "\n" + "[OUTPUT]\n" + "    Name              stdout\n" + "    Match             " + tagName(cluster, "*") + "\n" + "\n";
    Map<String, String> data = ImmutableMap.of("parsers.conf", parsersConfigFile, "fluentbit.conf", fluentBitConfigFile);
    ConfigMap configMap = new ConfigMapBuilder().withNewMetadata().withNamespace(namespace).withName(configName(context)).withLabels(labelFactory.clusterLabels(cluster)).endMetadata().withData(data).build();
    return Optional.of(configMap);
}

Example 28 with StackGresCluster

use of io.stackgres.common.crd.sgcluster.StackGresCluster in project stackgres by ongres.

the class FluentBit method buildSource.

protected Optional<HasMetadata> buildSource(StackGresClusterContext context) {
    final StackGresCluster cluster = context.getSource();
    if (cluster.getSpec().getDistributedLogs() == null) {
        return Optional.empty();
    }
    final String namespace = cluster.getMetadata().getNamespace();
    final String fluentdRelativeId = cluster.getSpec().getDistributedLogs().getDistributedLogs();
    final String fluentdNamespace = StackGresUtil.getNamespaceFromRelativeId(fluentdRelativeId, namespace);
    final String fluentdServiceName = FluentdUtil.serviceName(StackGresUtil.getNameFromRelativeId(fluentdRelativeId));
    String parsersConfigFile = "" + "[PARSER]\n" + "    Name        postgreslog_firstline\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2}.\\d*\\s\\S{3})" + ",(?<message>.*)\n" + "\n" + "[PARSER]\n" + "    Name        postgreslog_1\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2}.\\d*\\s\\S{3})" + ",(?<message>\"([^\"]*(?:\"\"[^\"]*)*)\"|)\n" + "\n" + "[PARSER]\n" + "    Name        patronilog_firstline\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2},\\d*{3})" + " (?<error_severity>[^:]+): (?<message>.*)\n" + "\n" + "[PARSER]\n" + "    Name        patronilog_1\n" + "    Format      regex\n" + "    Regex       " + "^(?<log_time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{2}:\\d{2}:\\d{2},\\d*{3})" + " (?<error_severity>[^:]+): (?<message>.*)\n" + "\n" + "[PARSER]\n" + "    Name        kubernetes_tag\n" + "    Format      regex\n" + "    Regex       ^[^.]+\\.[^.]+\\.[^.]+\\." + "(?<namespace_name>[^.]+)\\.(?<pod_name>[^.]+)$\n" + "\n";
    final String clusterNamespace = cluster.getMetadata().getNamespace();
    String fluentBitConfigFile = "" + "[SERVICE]\n" + "    Parsers_File      /etc/fluent-bit/parsers.conf\n" + "\n" + "[INPUT]\n" + "    Name              tail\n" + "    Path              " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/postgres*.csv\n" + "    Tag               " + FluentdUtil.POSTGRES_LOG_TYPE + "\n" + "    DB                " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/postgreslog.db\n" + "    Multiline         On\n" + "    Parser_Firstline  postgreslog_firstline\n" + "    Parser_1          postgreslog_1\n" + "    Buffer_Max_Size   2M\n" + "    Skip_Long_Lines   On\n" + "\n" + "[INPUT]\n" + "    Name              tail\n" + "    Key               message\n" + "    Path              " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/patroni*.log\n" + "    Tag               " + FluentdUtil.PATRONI_LOG_TYPE + "\n" + "    DB                " + ClusterStatefulSetPath.PG_LOG_PATH.path() + "/patronilog.db\n" + "    Multiline         On\n" + "    Parser_Firstline  patronilog_firstline\n" + "    Parser_1          patronilog_1\n" + "    Buffer_Max_Size   2M\n" + "    Skip_Long_Lines   On\n" + "\n" + "[FILTER]\n" + "    Name         rewrite_tag\n" + "    Match        " + FluentdUtil.POSTGRES_LOG_TYPE + "\n" + "    Rule         $message ^.*$ " + tagName(cluster, FluentdUtil.POSTGRES_LOG_TYPE) + "." + clusterNamespace + ".${HOSTNAME} false\n" + "    Emitter_Name postgres_re_emitted" + "\n" + "[FILTER]\n" + "    Name         rewrite_tag\n" + "    Match        " + FluentdUtil.PATRONI_LOG_TYPE + "\n" + "    Rule         $message ^.*$ " + tagName(cluster, FluentdUtil.PATRONI_LOG_TYPE) + "." + clusterNamespace + ".${HOSTNAME} false\n" + "    Emitter_Name patroni_re_emitted" + "\n" + "[FILTER]\n" + "    Name             kubernetes\n" + "    Match            " + tagName(cluster, "*") + "\n" + "    Annotations      Off\n" + "    Kube_Tag_Prefix  ''\n" + "    Regex_Parser     kubernetes_tag\n" + "    Buffer_Size      0\n" + "\n" + "[OUTPUT]\n" + "    Name              forward\n" + "    Match             " + tagName(cluster, "*") + "\n" + "    Host              " + fluentdServiceName + "." + fluentdNamespace + "\n" + "    Port              " + FluentdUtil.FORWARD_PORT + "\n" + "\n" + "[OUTPUT]\n" + "    Name              stdout\n" + "    Match             " + tagName(cluster, "*") + "\n" + "\n";
    Map<String, String> data = ImmutableMap.of("parsers.conf", parsersConfigFile, "fluentbit.conf", fluentBitConfigFile);
    ConfigMap configMap = new ConfigMapBuilder().withNewMetadata().withNamespace(namespace).withName(configName(context)).withLabels(labelFactory.clusterLabels(cluster)).endMetadata().withData(data).build();
    return Optional.of(configMap);
}

Example 29 with StackGresCluster

use of io.stackgres.common.crd.sgcluster.StackGresCluster in project stackgres by ongres.

the class PostgresExporter method getContainer.

@Override
public Container getContainer(StackGresClusterContainerContext context) {
    StackGresCluster cluster = context.getClusterContext().getSource();
    ContainerBuilder container = new ContainerBuilder();
    container.withName(NAME).withImage(StackGresComponent.PROMETHEUS_POSTGRES_EXPORTER.findLatestImageName()).withImagePullPolicy("IfNotPresent").withCommand("/bin/sh", "-exc").withArgs("" + "run_postgres_exporter() {\n" + "  set -x\n" + "  exec /usr/local/bin/postgres_exporter \\\n" + "    --log.level=" + (POSTGRES_EXPORTER_LOGGER.isTraceEnabled() ? "debug" : "info") + "\n" + "}\n" + "\n" + "set +x\n" + "while true\n" + "do\n" + "  if ( [ -z \"$PID\" ] || [ ! -d \"/proc/$PID\" ] ) \\\n" + "    && [ -S '" + ClusterStatefulSetPath.PG_RUN_PATH.path() + "/.s.PGSQL." + EnvoyUtil.PG_PORT + "' ]\n" + "  then\n" + "    if [ -n \"$PID\" ]\n" + "    then\n" + "      kill \"$PID\"\n" + "      wait \"$PID\" || true\n" + "    fi\n" + "    run_postgres_exporter &\n" + "    PID=\"$!\"\n" + "  fi\n" + "  sleep 5\n" + "done\n").withEnv(new EnvVarBuilder().withName("PGAPPNAME").withValue(NAME).build(), new EnvVarBuilder().withName("DATA_SOURCE_NAME").withValue("postgresql://postgres@:" + EnvoyUtil.PG_PORT + "/postgres" + "?host=" + ClusterStatefulSetPath.PG_RUN_PATH.path() + "&sslmode=disable").build(), new EnvVarBuilder().withName("PG_EXPORTER_EXTEND_QUERY_PATH").withValue("/var/opt/postgres-exporter/queries.yaml").build(), new EnvVarBuilder().withName("PG_EXPORTER_CONSTANT_LABELS").withValue("cluster_name=" + cluster.getMetadata().getName() + ", namespace=" + cluster.getMetadata().getNamespace()).build()).withPorts(new ContainerPortBuilder().withProtocol("TCP").withContainerPort(9187).build()).addAllToVolumeMounts(postgresSocket.getVolumeMounts(context)).addToVolumeMounts(new VolumeMountBuilder().withName(StatefulSetDynamicVolumes.EXPORTER_QUERIES.getVolumeName()).withMountPath("/var/opt/postgres-exporter/queries.yaml").withSubPath("queries.yaml").withReadOnly(true).build()).addAllToVolumeMounts(containerUserOverrideMounts.getVolumeMounts(context));
    return container.build();
}

Example 30 with StackGresCluster

use of io.stackgres.common.crd.sgcluster.StackGresCluster in project stackgres by ongres.

the class PrometheusIntegration method generateResource.

@Override
public Stream<HasMetadata> generateResource(StackGresClusterContext context) {
    final StackGresCluster cluster = context.getSource();
    final Map<String, String> defaultLabels = labelFactory.clusterLabels(cluster);
    Map<String, String> labels = new ImmutableMap.Builder<String, String>().putAll(labelFactory.clusterCrossNamespaceLabels(cluster)).build();
    final String clusterNamespace = cluster.getMetadata().getNamespace();
    final Stream<HasMetadata> resources = Stream.of(new ServiceBuilder().withNewMetadata().withNamespace(clusterNamespace).withName(serviceName(context)).withLabels(ImmutableMap.<String, String>builder().putAll(labels).put("container", POSTGRES_EXPORTER_CONTAINER_NAME).build()).endMetadata().withSpec(new ServiceSpecBuilder().withSelector(defaultLabels).withPorts(new ServicePortBuilder().withName(POSTGRES_EXPORTER_CONTAINER_NAME).withProtocol("TCP").withPort(9187).build()).build()).build());
    Optional<Stream<HasMetadata>> serviceMonitors = context.getPrometheus().filter(c -> Optional.ofNullable(c.getCreateServiceMonitor()).orElse(false)).map(c -> c.getPrometheusInstallations().stream().map(pi -> {
        ServiceMonitor serviceMonitor = new ServiceMonitor();
        serviceMonitor.setMetadata(new ObjectMetaBuilder().withNamespace(pi.getNamespace()).withName(serviceMonitorName(context)).withLabels(ImmutableMap.<String, String>builder().putAll(pi.getMatchLabels()).putAll(labels).build()).build());
        ServiceMonitorSpec spec = new ServiceMonitorSpec();
        serviceMonitor.setSpec(spec);
        LabelSelector selector = new LabelSelector();
        spec.setSelector(selector);
        NamespaceSelector namespaceSelector = new NamespaceSelector();
        namespaceSelector.setMatchNames(ImmutableList.of(clusterNamespace));
        spec.setNamespaceSelector(namespaceSelector);
        selector.setMatchLabels(labels);
        Endpoint endpoint = new Endpoint();
        endpoint.setPort(POSTGRES_EXPORTER_CONTAINER_NAME);
        spec.setEndpoints(Collections.singletonList(endpoint));
        return serviceMonitor;
    }));
    return serviceMonitors.map(hasMetadataStream -> Stream.concat(resources, hasMetadataStream)).orElse(resources);
}