Examples with ReportableConfig com.wavefront.agent.config.ReportableConfig used on opensource projects

Example 1 with ReportableConfig

use of com.wavefront.agent.config.ReportableConfig in project java by wavefrontHQ.

the class ProxyConfig method verifyAndInit.

@Override
public void verifyAndInit() {
    if (unparsed_params != null) {
        logger.info("Unparsed arguments: " + Joiner.on(", ").join(unparsed_params));
    }
    ReportableConfig config;
    // If they've specified a push configuration file, override the command line values
    try {
        if (pushConfigFile != null) {
            config = new ReportableConfig(pushConfigFile);
        } else {
            // dummy config
            config = new ReportableConfig();
        }
        prefix = Strings.emptyToNull(config.getString("prefix", prefix));
        // don't track token in proxy config metrics
        token = ObjectUtils.firstNonNull(config.getRawProperty("token", token), "undefined").trim();
        server = config.getString("server", server);
        hostname = config.getString("hostname", hostname);
        idFile = config.getString("idFile", idFile);
        pushRateLimit = config.getInteger("pushRateLimit", pushRateLimit);
        pushRateLimitHistograms = config.getInteger("pushRateLimitHistograms", pushRateLimitHistograms);
        pushRateLimitSourceTags = config.getDouble("pushRateLimitSourceTags", pushRateLimitSourceTags);
        pushRateLimitSpans = config.getInteger("pushRateLimitSpans", pushRateLimitSpans);
        pushRateLimitSpanLogs = config.getInteger("pushRateLimitSpanLogs", pushRateLimitSpanLogs);
        pushRateLimitEvents = config.getDouble("pushRateLimitEvents", pushRateLimitEvents);
        pushRateLimitMaxBurstSeconds = config.getInteger("pushRateLimitMaxBurstSeconds", pushRateLimitMaxBurstSeconds);
        pushBlockedSamples = config.getInteger("pushBlockedSamples", pushBlockedSamples);
        blockedPointsLoggerName = config.getString("blockedPointsLoggerName", blockedPointsLoggerName);
        blockedHistogramsLoggerName = config.getString("blockedHistogramsLoggerName", blockedHistogramsLoggerName);
        blockedSpansLoggerName = config.getString("blockedSpansLoggerName", blockedSpansLoggerName);
        pushListenerPorts = config.getString("pushListenerPorts", pushListenerPorts);
        pushListenerMaxReceivedLength = config.getInteger("pushListenerMaxReceivedLength", pushListenerMaxReceivedLength);
        pushListenerHttpBufferSize = config.getInteger("pushListenerHttpBufferSize", pushListenerHttpBufferSize);
        traceListenerMaxReceivedLength = config.getInteger("traceListenerMaxReceivedLength", traceListenerMaxReceivedLength);
        traceListenerHttpBufferSize = config.getInteger("traceListenerHttpBufferSize", traceListenerHttpBufferSize);
        listenerIdleConnectionTimeout = config.getInteger("listenerIdleConnectionTimeout", listenerIdleConnectionTimeout);
        memGuardFlushThreshold = config.getInteger("memGuardFlushThreshold", memGuardFlushThreshold);
        // Histogram: global settings
        histogramPassthroughRecompression = config.getBoolean("histogramPassthroughRecompression", histogramPassthroughRecompression);
        histogramStateDirectory = config.getString("histogramStateDirectory", histogramStateDirectory);
        histogramAccumulatorResolveInterval = config.getLong("histogramAccumulatorResolveInterval", histogramAccumulatorResolveInterval);
        histogramAccumulatorFlushInterval = config.getLong("histogramAccumulatorFlushInterval", histogramAccumulatorFlushInterval);
        histogramAccumulatorFlushMaxBatchSize = config.getInteger("histogramAccumulatorFlushMaxBatchSize", histogramAccumulatorFlushMaxBatchSize);
        histogramMaxReceivedLength = config.getInteger("histogramMaxReceivedLength", histogramMaxReceivedLength);
        histogramHttpBufferSize = config.getInteger("histogramHttpBufferSize", histogramHttpBufferSize);
        deltaCountersAggregationListenerPorts = config.getString("deltaCountersAggregationListenerPorts", deltaCountersAggregationListenerPorts);
        deltaCountersAggregationIntervalSeconds = config.getLong("deltaCountersAggregationIntervalSeconds", deltaCountersAggregationIntervalSeconds);
        customTracingListenerPorts = config.getString("customTracingListenerPorts", customTracingListenerPorts);
        // Histogram: deprecated settings - fall back for backwards compatibility
        if (config.isDefined("avgHistogramKeyBytes")) {
            histogramMinuteAvgKeyBytes = histogramHourAvgKeyBytes = histogramDayAvgKeyBytes = histogramDistAvgKeyBytes = config.getInteger("avgHistogramKeyBytes", 150);
        }
        if (config.isDefined("avgHistogramDigestBytes")) {
            histogramMinuteAvgDigestBytes = histogramHourAvgDigestBytes = histogramDayAvgDigestBytes = histogramDistAvgDigestBytes = config.getInteger("avgHistogramDigestBytes", 500);
        }
        if (config.isDefined("histogramAccumulatorSize")) {
            histogramMinuteAccumulatorSize = histogramHourAccumulatorSize = histogramDayAccumulatorSize = histogramDistAccumulatorSize = config.getLong("histogramAccumulatorSize", 100000);
        }
        if (config.isDefined("histogramCompression")) {
            histogramMinuteCompression = histogramHourCompression = histogramDayCompression = histogramDistCompression = config.getNumber("histogramCompression", null, 20, 1000).shortValue();
        }
        if (config.isDefined("persistAccumulator")) {
            histogramMinuteAccumulatorPersisted = histogramHourAccumulatorPersisted = histogramDayAccumulatorPersisted = histogramDistAccumulatorPersisted = config.getBoolean("persistAccumulator", false);
        }
        // Histogram: minute accumulator settings
        histogramMinuteListenerPorts = config.getString("histogramMinuteListenerPorts", histogramMinuteListenerPorts);
        histogramMinuteFlushSecs = config.getInteger("histogramMinuteFlushSecs", histogramMinuteFlushSecs);
        histogramMinuteCompression = config.getNumber("histogramMinuteCompression", histogramMinuteCompression, 20, 1000).shortValue();
        histogramMinuteAvgKeyBytes = config.getInteger("histogramMinuteAvgKeyBytes", histogramMinuteAvgKeyBytes);
        histogramMinuteAvgDigestBytes = 32 + histogramMinuteCompression * 7;
        histogramMinuteAvgDigestBytes = config.getInteger("histogramMinuteAvgDigestBytes", histogramMinuteAvgDigestBytes);
        histogramMinuteAccumulatorSize = config.getLong("histogramMinuteAccumulatorSize", histogramMinuteAccumulatorSize);
        histogramMinuteAccumulatorPersisted = config.getBoolean("histogramMinuteAccumulatorPersisted", histogramMinuteAccumulatorPersisted);
        histogramMinuteMemoryCache = config.getBoolean("histogramMinuteMemoryCache", histogramMinuteMemoryCache);
        // Histogram: hour accumulator settings
        histogramHourListenerPorts = config.getString("histogramHourListenerPorts", histogramHourListenerPorts);
        histogramHourFlushSecs = config.getInteger("histogramHourFlushSecs", histogramHourFlushSecs);
        histogramHourCompression = config.getNumber("histogramHourCompression", histogramHourCompression, 20, 1000).shortValue();
        histogramHourAvgKeyBytes = config.getInteger("histogramHourAvgKeyBytes", histogramHourAvgKeyBytes);
        histogramHourAvgDigestBytes = 32 + histogramHourCompression * 7;
        histogramHourAvgDigestBytes = config.getInteger("histogramHourAvgDigestBytes", histogramHourAvgDigestBytes);
        histogramHourAccumulatorSize = config.getLong("histogramHourAccumulatorSize", histogramHourAccumulatorSize);
        histogramHourAccumulatorPersisted = config.getBoolean("histogramHourAccumulatorPersisted", histogramHourAccumulatorPersisted);
        histogramHourMemoryCache = config.getBoolean("histogramHourMemoryCache", histogramHourMemoryCache);
        // Histogram: day accumulator settings
        histogramDayListenerPorts = config.getString("histogramDayListenerPorts", histogramDayListenerPorts);
        histogramDayFlushSecs = config.getInteger("histogramDayFlushSecs", histogramDayFlushSecs);
        histogramDayCompression = config.getNumber("histogramDayCompression", histogramDayCompression, 20, 1000).shortValue();
        histogramDayAvgKeyBytes = config.getInteger("histogramDayAvgKeyBytes", histogramDayAvgKeyBytes);
        histogramDayAvgDigestBytes = 32 + histogramDayCompression * 7;
        histogramDayAvgDigestBytes = config.getInteger("histogramDayAvgDigestBytes", histogramDayAvgDigestBytes);
        histogramDayAccumulatorSize = config.getLong("histogramDayAccumulatorSize", histogramDayAccumulatorSize);
        histogramDayAccumulatorPersisted = config.getBoolean("histogramDayAccumulatorPersisted", histogramDayAccumulatorPersisted);
        histogramDayMemoryCache = config.getBoolean("histogramDayMemoryCache", histogramDayMemoryCache);
        // Histogram: dist accumulator settings
        histogramDistListenerPorts = config.getString("histogramDistListenerPorts", histogramDistListenerPorts);
        histogramDistFlushSecs = config.getInteger("histogramDistFlushSecs", histogramDistFlushSecs);
        histogramDistCompression = config.getNumber("histogramDistCompression", histogramDistCompression, 20, 1000).shortValue();
        histogramDistAvgKeyBytes = config.getInteger("histogramDistAvgKeyBytes", histogramDistAvgKeyBytes);
        histogramDistAvgDigestBytes = 32 + histogramDistCompression * 7;
        histogramDistAvgDigestBytes = config.getInteger("histogramDistAvgDigestBytes", histogramDistAvgDigestBytes);
        histogramDistAccumulatorSize = config.getLong("histogramDistAccumulatorSize", histogramDistAccumulatorSize);
        histogramDistAccumulatorPersisted = config.getBoolean("histogramDistAccumulatorPersisted", histogramDistAccumulatorPersisted);
        histogramDistMemoryCache = config.getBoolean("histogramDistMemoryCache", histogramDistMemoryCache);
        exportQueuePorts = config.getString("exportQueuePorts", exportQueuePorts);
        exportQueueOutputFile = config.getString("exportQueueOutputFile", exportQueueOutputFile);
        exportQueueRetainData = config.getBoolean("exportQueueRetainData", exportQueueRetainData);
        useNoopSender = config.getBoolean("useNoopSender", useNoopSender);
        flushThreads = config.getInteger("flushThreads", flushThreads);
        flushThreadsEvents = config.getInteger("flushThreadsEvents", flushThreadsEvents);
        flushThreadsSourceTags = config.getInteger("flushThreadsSourceTags", flushThreadsSourceTags);
        jsonListenerPorts = config.getString("jsonListenerPorts", jsonListenerPorts);
        writeHttpJsonListenerPorts = config.getString("writeHttpJsonListenerPorts", writeHttpJsonListenerPorts);
        dataDogJsonPorts = config.getString("dataDogJsonPorts", dataDogJsonPorts);
        dataDogRequestRelayTarget = config.getString("dataDogRequestRelayTarget", dataDogRequestRelayTarget);
        dataDogRequestRelayAsyncThreads = config.getInteger("dataDogRequestRelayAsyncThreads", dataDogRequestRelayAsyncThreads);
        dataDogRequestRelaySyncMode = config.getBoolean("dataDogRequestRelaySyncMode", dataDogRequestRelaySyncMode);
        dataDogProcessSystemMetrics = config.getBoolean("dataDogProcessSystemMetrics", dataDogProcessSystemMetrics);
        dataDogProcessServiceChecks = config.getBoolean("dataDogProcessServiceChecks", dataDogProcessServiceChecks);
        graphitePorts = config.getString("graphitePorts", graphitePorts);
        graphiteFormat = config.getString("graphiteFormat", graphiteFormat);
        graphiteFieldsToRemove = config.getString("graphiteFieldsToRemove", graphiteFieldsToRemove);
        graphiteDelimiters = config.getString("graphiteDelimiters", graphiteDelimiters);
        allowRegex = config.getString("allowRegex", config.getString("whitelistRegex", allowRegex));
        blockRegex = config.getString("blockRegex", config.getString("blacklistRegex", blockRegex));
        opentsdbPorts = config.getString("opentsdbPorts", opentsdbPorts);
        opentsdbAllowRegex = config.getString("opentsdbAllowRegex", config.getString("opentsdbWhitelistRegex", opentsdbAllowRegex));
        opentsdbBlockRegex = config.getString("opentsdbBlockRegex", config.getString("opentsdbBlacklistRegex", opentsdbBlockRegex));
        proxyHost = config.getString("proxyHost", proxyHost);
        proxyPort = config.getInteger("proxyPort", proxyPort);
        proxyPassword = config.getString("proxyPassword", proxyPassword, s -> "<removed>");
        proxyUser = config.getString("proxyUser", proxyUser);
        httpUserAgent = config.getString("httpUserAgent", httpUserAgent);
        httpConnectTimeout = config.getInteger("httpConnectTimeout", httpConnectTimeout);
        httpRequestTimeout = config.getInteger("httpRequestTimeout", httpRequestTimeout);
        httpMaxConnTotal = Math.min(200, config.getInteger("httpMaxConnTotal", httpMaxConnTotal));
        httpMaxConnPerRoute = Math.min(100, config.getInteger("httpMaxConnPerRoute", httpMaxConnPerRoute));
        httpAutoRetries = config.getInteger("httpAutoRetries", httpAutoRetries);
        gzipCompression = config.getBoolean("gzipCompression", gzipCompression);
        gzipCompressionLevel = config.getNumber("gzipCompressionLevel", gzipCompressionLevel, 1, 9).intValue();
        soLingerTime = config.getInteger("soLingerTime", soLingerTime);
        splitPushWhenRateLimited = config.getBoolean("splitPushWhenRateLimited", splitPushWhenRateLimited);
        customSourceTags = config.getString("customSourceTags", customSourceTags);
        agentMetricsPointTags = config.getString("agentMetricsPointTags", agentMetricsPointTags);
        ephemeral = config.getBoolean("ephemeral", ephemeral);
        disableRdnsLookup = config.getBoolean("disableRdnsLookup", disableRdnsLookup);
        picklePorts = config.getString("picklePorts", picklePorts);
        traceListenerPorts = config.getString("traceListenerPorts", traceListenerPorts);
        traceJaegerListenerPorts = config.getString("traceJaegerListenerPorts", traceJaegerListenerPorts);
        traceJaegerHttpListenerPorts = config.getString("traceJaegerHttpListenerPorts", traceJaegerHttpListenerPorts);
        traceJaegerGrpcListenerPorts = config.getString("traceJaegerGrpcListenerPorts", traceJaegerGrpcListenerPorts);
        traceJaegerApplicationName = config.getString("traceJaegerApplicationName", traceJaegerApplicationName);
        traceZipkinListenerPorts = config.getString("traceZipkinListenerPorts", traceZipkinListenerPorts);
        traceZipkinApplicationName = config.getString("traceZipkinApplicationName", traceZipkinApplicationName);
        customTracingListenerPorts = config.getString("customTracingListenerPorts", customTracingListenerPorts);
        customTracingApplicationName = config.getString("customTracingApplicationName", customTracingApplicationName);
        customTracingServiceName = config.getString("customTracingServiceName", customTracingServiceName);
        traceSamplingRate = config.getDouble("traceSamplingRate", traceSamplingRate);
        traceSamplingDuration = config.getInteger("traceSamplingDuration", traceSamplingDuration);
        traceDerivedCustomTagKeys = config.getString("traceDerivedCustomTagKeys", traceDerivedCustomTagKeys);
        backendSpanHeadSamplingPercentIgnored = config.getBoolean("backendSpanHeadSamplingPercentIgnored", backendSpanHeadSamplingPercentIgnored);
        pushRelayListenerPorts = config.getString("pushRelayListenerPorts", pushRelayListenerPorts);
        pushRelayHistogramAggregator = config.getBoolean("pushRelayHistogramAggregator", pushRelayHistogramAggregator);
        pushRelayHistogramAggregatorAccumulatorSize = config.getLong("pushRelayHistogramAggregatorAccumulatorSize", pushRelayHistogramAggregatorAccumulatorSize);
        pushRelayHistogramAggregatorFlushSecs = config.getInteger("pushRelayHistogramAggregatorFlushSecs", pushRelayHistogramAggregatorFlushSecs);
        pushRelayHistogramAggregatorCompression = config.getNumber("pushRelayHistogramAggregatorCompression", pushRelayHistogramAggregatorCompression).shortValue();
        bufferFile = config.getString("buffer", bufferFile);
        bufferShardSize = config.getInteger("bufferShardSize", bufferShardSize);
        disableBufferSharding = config.getBoolean("disableBufferSharding", disableBufferSharding);
        taskQueueLevel = TaskQueueLevel.fromString(config.getString("taskQueueStrategy", taskQueueLevel.toString()));
        purgeBuffer = config.getBoolean("purgeBuffer", purgeBuffer);
        preprocessorConfigFile = config.getString("preprocessorConfigFile", preprocessorConfigFile);
        dataBackfillCutoffHours = config.getInteger("dataBackfillCutoffHours", dataBackfillCutoffHours);
        dataPrefillCutoffHours = config.getInteger("dataPrefillCutoffHours", dataPrefillCutoffHours);
        filebeatPort = config.getInteger("filebeatPort", filebeatPort);
        rawLogsPort = config.getInteger("rawLogsPort", rawLogsPort);
        rawLogsMaxReceivedLength = config.getInteger("rawLogsMaxReceivedLength", rawLogsMaxReceivedLength);
        rawLogsHttpBufferSize = config.getInteger("rawLogsHttpBufferSize", rawLogsHttpBufferSize);
        logsIngestionConfigFile = config.getString("logsIngestionConfigFile", logsIngestionConfigFile);
        sqsQueueBuffer = config.getBoolean("sqsBuffer", sqsQueueBuffer);
        sqsQueueNameTemplate = config.getString("sqsQueueNameTemplate", sqsQueueNameTemplate);
        sqsQueueRegion = config.getString("sqsQueueRegion", sqsQueueRegion);
        sqsQueueIdentifier = config.getString("sqsQueueIdentifier", sqsQueueIdentifier);
        // auth settings
        authMethod = TokenValidationMethod.fromString(config.getString("authMethod", authMethod.toString()));
        authTokenIntrospectionServiceUrl = config.getString("authTokenIntrospectionServiceUrl", authTokenIntrospectionServiceUrl);
        authTokenIntrospectionAuthorizationHeader = config.getString("authTokenIntrospectionAuthorizationHeader", authTokenIntrospectionAuthorizationHeader);
        authResponseRefreshInterval = config.getInteger("authResponseRefreshInterval", authResponseRefreshInterval);
        authResponseMaxTtl = config.getInteger("authResponseMaxTtl", authResponseMaxTtl);
        authStaticToken = config.getString("authStaticToken", authStaticToken);
        // health check / admin API settings
        adminApiListenerPort = config.getInteger("adminApiListenerPort", adminApiListenerPort);
        adminApiRemoteIpAllowRegex = config.getString("adminApiRemoteIpWhitelistRegex", adminApiRemoteIpAllowRegex);
        httpHealthCheckPorts = config.getString("httpHealthCheckPorts", httpHealthCheckPorts);
        httpHealthCheckAllPorts = config.getBoolean("httpHealthCheckAllPorts", false);
        httpHealthCheckPath = config.getString("httpHealthCheckPath", httpHealthCheckPath);
        httpHealthCheckResponseContentType = config.getString("httpHealthCheckResponseContentType", httpHealthCheckResponseContentType);
        httpHealthCheckPassStatusCode = config.getInteger("httpHealthCheckPassStatusCode", httpHealthCheckPassStatusCode);
        httpHealthCheckPassResponseBody = config.getString("httpHealthCheckPassResponseBody", httpHealthCheckPassResponseBody);
        httpHealthCheckFailStatusCode = config.getInteger("httpHealthCheckFailStatusCode", httpHealthCheckFailStatusCode);
        httpHealthCheckFailResponseBody = config.getString("httpHealthCheckFailResponseBody", httpHealthCheckFailResponseBody);
        // TLS configurations
        privateCertPath = config.getString("privateCertPath", privateCertPath);
        privateKeyPath = config.getString("privateKeyPath", privateKeyPath);
        tlsPorts = config.getString("tlsPorts", tlsPorts);
        // Traffic shaping config
        trafficShaping = config.getBoolean("trafficShaping", trafficShaping);
        trafficShapingWindowSeconds = config.getInteger("trafficShapingWindowSeconds", trafficShapingWindowSeconds);
        trafficShapingHeadroom = config.getDouble("trafficShapingHeadroom", trafficShapingHeadroom);
        // CORS configuration
        corsEnabledPorts = config.getString("corsEnabledPorts", corsEnabledPorts);
        corsOrigin = config.getString("corsOrigin", corsOrigin);
        corsAllowNullOrigin = config.getBoolean("corsAllowNullOrigin", corsAllowNullOrigin);
        // clamp values for pushFlushMaxPoints/etc between min split size
        // (or 1 in case of source tags and events) and default batch size.
        // also make sure it is never higher than the configured rate limit.
        pushFlushMaxPoints = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxPoints", pushFlushMaxPoints), DEFAULT_BATCH_SIZE), (int) pushRateLimit), DEFAULT_MIN_SPLIT_BATCH_SIZE);
        pushFlushMaxHistograms = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxHistograms", pushFlushMaxHistograms), DEFAULT_BATCH_SIZE_HISTOGRAMS), (int) pushRateLimitHistograms), DEFAULT_MIN_SPLIT_BATCH_SIZE);
        pushFlushMaxSourceTags = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxSourceTags", pushFlushMaxSourceTags), DEFAULT_BATCH_SIZE_SOURCE_TAGS), (int) pushRateLimitSourceTags), 1);
        pushFlushMaxSpans = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxSpans", pushFlushMaxSpans), DEFAULT_BATCH_SIZE_SPANS), (int) pushRateLimitSpans), DEFAULT_MIN_SPLIT_BATCH_SIZE);
        pushFlushMaxSpanLogs = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxSpanLogs", pushFlushMaxSpanLogs), DEFAULT_BATCH_SIZE_SPAN_LOGS), (int) pushRateLimitSpanLogs), DEFAULT_MIN_SPLIT_BATCH_SIZE);
        pushFlushMaxEvents = Math.min(Math.min(Math.max(config.getInteger("pushFlushMaxEvents", pushFlushMaxEvents), 1), DEFAULT_BATCH_SIZE_EVENTS), (int) (pushRateLimitEvents + 1));
        /*
        default value for pushMemoryBufferLimit is 16 * pushFlushMaxPoints, but no more than 25% of
        available heap memory. 25% is chosen heuristically as a safe number for scenarios with
        limited system resources (4 CPU cores or less, heap size less than 4GB) to prevent OOM.
        this is a conservative estimate, budgeting 200 characters (400 bytes) per per point line.
        Also, it shouldn't be less than 1 batch size (pushFlushMaxPoints).
       */
        int listeningPorts = Iterables.size(Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts));
        long calculatedMemoryBufferLimit = Math.max(Math.min(16 * pushFlushMaxPoints, Runtime.getRuntime().maxMemory() / Math.max(0, listeningPorts) / 4 / flushThreads / 400), pushFlushMaxPoints);
        logger.fine("Calculated pushMemoryBufferLimit: " + calculatedMemoryBufferLimit);
        pushMemoryBufferLimit = Math.max(config.getInteger("pushMemoryBufferLimit", pushMemoryBufferLimit), pushFlushMaxPoints);
        logger.fine("Configured pushMemoryBufferLimit: " + pushMemoryBufferLimit);
        pushFlushInterval = config.getInteger("pushFlushInterval", pushFlushInterval);
        retryBackoffBaseSeconds = Math.max(Math.min(config.getDouble("retryBackoffBaseSeconds", retryBackoffBaseSeconds), MAX_RETRY_BACKOFF_BASE_SECONDS), 1.0);
    } catch (Throwable exception) {
        logger.severe("Could not load configuration file " + pushConfigFile);
        throw new RuntimeException(exception.getMessage());
    }
    if (httpUserAgent == null) {
        httpUserAgent = "Wavefront-Proxy/" + getBuildVersion();
    }
    if (pushConfigFile != null) {
        logger.info("Loaded configuration file " + pushConfigFile);
    }
}

Example 2 with ReportableConfig

use of com.wavefront.agent.config.ReportableConfig in project java by wavefrontHQ.

the class AbstractAgent method loadListenerConfigurationFile.

private void loadListenerConfigurationFile() throws IOException {
    // If they've specified a push configuration file, override the command line values
    if (pushConfigFile != null) {
        ReportableConfig config = new ReportableConfig(pushConfigFile);
        try {
            prefix = Strings.emptyToNull(config.getString("prefix", prefix));
            pushLogLevel = config.getString("pushLogLevel", pushLogLevel);
            pushValidationLevel = config.getString("pushValidationLevel", pushValidationLevel);
            // don't track
            token = config.getRawProperty("token", token).trim();
            // don't track
            server = config.getRawProperty("server", server).trim();
            hostname = config.getString("hostname", hostname);
            idFile = config.getString("idFile", idFile);
            pushRateLimit = config.getNumber("pushRateLimit", pushRateLimit).intValue();
            pushRateLimitMaxBurstSeconds = config.getNumber("pushRateLimitMaxBurstSeconds", pushRateLimitMaxBurstSeconds).intValue();
            pushBlockedSamples = config.getNumber("pushBlockedSamples", pushBlockedSamples).intValue();
            pushListenerPorts = config.getString("pushListenerPorts", pushListenerPorts);
            memGuardFlushThreshold = config.getNumber("memGuardFlushThreshold", memGuardFlushThreshold).intValue();
            // Histogram: global settings
            histogramStateDirectory = config.getString("histogramStateDirectory", histogramStateDirectory);
            histogramAccumulatorResolveInterval = config.getNumber("histogramAccumulatorResolveInterval", histogramAccumulatorResolveInterval).longValue();
            histogramAccumulatorFlushInterval = config.getNumber("histogramAccumulatorFlushInterval", histogramAccumulatorFlushInterval).longValue();
            histogramAccumulatorFlushMaxBatchSize = config.getNumber("histogramAccumulatorFlushMaxBatchSize", histogramAccumulatorFlushMaxBatchSize).intValue();
            histogramReceiveBufferFlushInterval = config.getNumber("histogramReceiveBufferFlushInterval", histogramReceiveBufferFlushInterval).intValue();
            histogramProcessingQueueScanInterval = config.getNumber("histogramProcessingQueueScanInterval", histogramProcessingQueueScanInterval).intValue();
            persistAccumulator = config.getBoolean("persistAccumulator", persistAccumulator);
            persistMessages = config.getBoolean("persistMessages", persistMessages);
            persistMessagesCompression = config.getBoolean("persistMessagesCompression", persistMessagesCompression);
            // Histogram: deprecated settings - fall back for backwards compatibility
            if (config.isDefined("avgHistogramKeyBytes")) {
                histogramMinuteAvgKeyBytes = histogramHourAvgKeyBytes = histogramDayAvgKeyBytes = histogramDistAvgKeyBytes = config.getNumber("avgHistogramKeyBytes", avgHistogramKeyBytes).intValue();
            }
            if (config.isDefined("avgHistogramDigestBytes")) {
                histogramMinuteAvgDigestBytes = histogramHourAvgDigestBytes = histogramDayAvgDigestBytes = histogramDistAvgDigestBytes = config.getNumber("avgHistogramDigestBytes", avgHistogramDigestBytes).intValue();
            }
            if (config.isDefined("histogramAccumulatorSize")) {
                histogramMinuteAccumulatorSize = histogramHourAccumulatorSize = histogramDayAccumulatorSize = histogramDistAccumulatorSize = config.getNumber("histogramAccumulatorSize", histogramAccumulatorSize).longValue();
            }
            if (config.isDefined("histogramCompression")) {
                histogramMinuteCompression = histogramHourCompression = histogramDayCompression = histogramDistCompression = config.getNumber("histogramCompression", null, 20, 1000).shortValue();
            }
            // Histogram: minute accumulator settings
            histogramMinuteListenerPorts = config.getString("histogramMinuteListenerPorts", histogramMinuteListenerPorts);
            histogramMinuteAccumulators = config.getNumber("histogramMinuteAccumulators", histogramMinuteAccumulators).intValue();
            histogramMinuteFlushSecs = config.getNumber("histogramMinuteFlushSecs", histogramMinuteFlushSecs).intValue();
            histogramMinuteCompression = config.getNumber("histogramMinuteCompression", histogramMinuteCompression, 20, 1000).shortValue();
            histogramMinuteAvgKeyBytes = config.getNumber("histogramMinuteAvgKeyBytes", histogramMinuteAvgKeyBytes).intValue();
            histogramMinuteAvgDigestBytes = 32 + histogramMinuteCompression * 7;
            histogramMinuteAvgDigestBytes = config.getNumber("histogramMinuteAvgDigestBytes", histogramMinuteAvgDigestBytes).intValue();
            histogramMinuteAccumulatorSize = config.getNumber("histogramMinuteAccumulatorSize", histogramMinuteAccumulatorSize).longValue();
            histogramMinuteMemoryCache = config.getBoolean("histogramMinuteMemoryCache", histogramMinuteMemoryCache);
            // Histogram: hour accumulator settings
            histogramHourListenerPorts = config.getString("histogramHourListenerPorts", histogramHourListenerPorts);
            histogramHourAccumulators = config.getNumber("histogramHourAccumulators", histogramHourAccumulators).intValue();
            histogramHourFlushSecs = config.getNumber("histogramHourFlushSecs", histogramHourFlushSecs).intValue();
            histogramHourCompression = config.getNumber("histogramHourCompression", histogramHourCompression, 20, 1000).shortValue();
            histogramHourAvgKeyBytes = config.getNumber("histogramHourAvgKeyBytes", histogramHourAvgKeyBytes).intValue();
            histogramHourAvgDigestBytes = 32 + histogramHourCompression * 7;
            histogramHourAvgDigestBytes = config.getNumber("histogramHourAvgDigestBytes", histogramHourAvgDigestBytes).intValue();
            histogramHourAccumulatorSize = config.getNumber("histogramHourAccumulatorSize", histogramHourAccumulatorSize).longValue();
            histogramHourMemoryCache = config.getBoolean("histogramHourMemoryCache", histogramHourMemoryCache);
            // Histogram: day accumulator settings
            histogramDayListenerPorts = config.getString("histogramDayListenerPorts", histogramDayListenerPorts);
            histogramDayAccumulators = config.getNumber("histogramDayAccumulators", histogramDayAccumulators).intValue();
            histogramDayFlushSecs = config.getNumber("histogramDayFlushSecs", histogramDayFlushSecs).intValue();
            histogramDayCompression = config.getNumber("histogramDayCompression", histogramDayCompression, 20, 1000).shortValue();
            histogramDayAvgKeyBytes = config.getNumber("histogramDayAvgKeyBytes", histogramDayAvgKeyBytes).intValue();
            histogramDayAvgDigestBytes = 32 + histogramDayCompression * 7;
            histogramDayAvgDigestBytes = config.getNumber("histogramDayAvgDigestBytes", histogramDayAvgDigestBytes).intValue();
            histogramDayAccumulatorSize = config.getNumber("histogramDayAccumulatorSize", histogramDayAccumulatorSize).longValue();
            histogramDayMemoryCache = config.getBoolean("histogramDayMemoryCache", histogramDayMemoryCache);
            // Histogram: dist accumulator settings
            histogramDistListenerPorts = config.getString("histogramDistListenerPorts", histogramDistListenerPorts);
            histogramDistAccumulators = config.getNumber("histogramDistAccumulators", histogramDistAccumulators).intValue();
            histogramDistFlushSecs = config.getNumber("histogramDistFlushSecs", histogramDistFlushSecs).intValue();
            histogramDistCompression = config.getNumber("histogramDistCompression", histogramDistCompression, 20, 1000).shortValue();
            histogramDistAvgKeyBytes = config.getNumber("histogramDistAvgKeyBytes", histogramDistAvgKeyBytes).intValue();
            histogramDistAvgDigestBytes = 32 + histogramDistCompression * 7;
            histogramDistAvgDigestBytes = config.getNumber("histogramDistAvgDigestBytes", histogramDistAvgDigestBytes).intValue();
            histogramDistAccumulatorSize = config.getNumber("histogramDistAccumulatorSize", histogramDistAccumulatorSize).longValue();
            histogramDistMemoryCache = config.getBoolean("histogramDistMemoryCache", histogramDistMemoryCache);
            retryThreads = config.getNumber("retryThreads", retryThreads).intValue();
            flushThreads = config.getNumber("flushThreads", flushThreads).intValue();
            httpJsonPorts = config.getString("jsonListenerPorts", httpJsonPorts);
            writeHttpJsonPorts = config.getString("writeHttpJsonListenerPorts", writeHttpJsonPorts);
            graphitePorts = config.getString("graphitePorts", graphitePorts);
            graphiteFormat = config.getString("graphiteFormat", graphiteFormat);
            graphiteFieldsToRemove = config.getString("graphiteFieldsToRemove", graphiteFieldsToRemove);
            graphiteDelimiters = config.getString("graphiteDelimiters", graphiteDelimiters);
            graphiteWhitelistRegex = config.getString("graphiteWhitelistRegex", graphiteWhitelistRegex);
            graphiteBlacklistRegex = config.getString("graphiteBlacklistRegex", graphiteBlacklistRegex);
            whitelistRegex = config.getString("whitelistRegex", whitelistRegex);
            blacklistRegex = config.getString("blacklistRegex", blacklistRegex);
            opentsdbPorts = config.getString("opentsdbPorts", opentsdbPorts);
            opentsdbWhitelistRegex = config.getString("opentsdbWhitelistRegex", opentsdbWhitelistRegex);
            opentsdbBlacklistRegex = config.getString("opentsdbBlacklistRegex", opentsdbBlacklistRegex);
            proxyHost = config.getString("proxyHost", proxyHost);
            proxyPort = config.getNumber("proxyPort", proxyPort).intValue();
            proxyPassword = config.getString("proxyPassword", proxyPassword, s -> "<removed>");
            proxyUser = config.getString("proxyUser", proxyUser);
            httpUserAgent = config.getString("httpUserAgent", httpUserAgent);
            httpConnectTimeout = config.getNumber("httpConnectTimeout", httpConnectTimeout).intValue();
            httpRequestTimeout = config.getNumber("httpRequestTimeout", httpRequestTimeout).intValue();
            httpMaxConnTotal = Math.min(200, config.getNumber("httpMaxConnTotal", httpMaxConnTotal).intValue());
            httpMaxConnPerRoute = Math.min(100, config.getNumber("httpMaxConnPerRoute", httpMaxConnPerRoute).intValue());
            httpAutoRetries = config.getNumber("httpAutoRetries", httpAutoRetries).intValue();
            javaNetConnection = config.getBoolean("javaNetConnection", javaNetConnection);
            gzipCompression = config.getBoolean("gzipCompression", gzipCompression);
            soLingerTime = config.getNumber("soLingerTime", soLingerTime).intValue();
            splitPushWhenRateLimited = config.getBoolean("splitPushWhenRateLimited", splitPushWhenRateLimited);
            customSourceTagsProperty = config.getString("customSourceTags", customSourceTagsProperty);
            agentMetricsPointTags = config.getString("agentMetricsPointTags", agentMetricsPointTags);
            ephemeral = config.getBoolean("ephemeral", ephemeral);
            disableRdnsLookup = config.getBoolean("disableRdnsLookup", disableRdnsLookup);
            picklePorts = config.getString("picklePorts", picklePorts);
            bufferFile = config.getString("buffer", bufferFile);
            preprocessorConfigFile = config.getString("preprocessorConfigFile", preprocessorConfigFile);
            dataBackfillCutoffHours = config.getNumber("dataBackfillCutoffHours", dataBackfillCutoffHours).intValue();
            dataPrefillCutoffHours = config.getNumber("dataPrefillCutoffHours", dataPrefillCutoffHours).intValue();
            filebeatPort = config.getNumber("filebeatPort", filebeatPort).intValue();
            rawLogsPort = config.getNumber("rawLogsPort", rawLogsPort).intValue();
            logsIngestionConfigFile = config.getString("logsIngestionConfigFile", logsIngestionConfigFile);
            // track mutable settings
            pushFlushIntervalInitialValue = Integer.parseInt(config.getRawProperty("pushFlushInterval", String.valueOf(pushFlushInterval.get())).trim());
            pushFlushInterval.set(pushFlushIntervalInitialValue);
            config.reportSettingAsGauge(pushFlushInterval, "pushFlushInterval");
            pushFlushMaxPointsInitialValue = Integer.parseInt(config.getRawProperty("pushFlushMaxPoints", String.valueOf(pushFlushMaxPoints.get())).trim());
            // clamp values for pushFlushMaxPoints between 1..50000
            pushFlushMaxPointsInitialValue = Math.max(Math.min(pushFlushMaxPointsInitialValue, MAX_SPLIT_BATCH_SIZE), 1);
            pushFlushMaxPoints.set(pushFlushMaxPointsInitialValue);
            config.reportSettingAsGauge(pushFlushMaxPoints, "pushFlushMaxPoints");
            retryBackoffBaseSecondsInitialValue = Double.parseDouble(config.getRawProperty("retryBackoffBaseSeconds", String.valueOf(retryBackoffBaseSeconds.get())).trim());
            retryBackoffBaseSeconds.set(retryBackoffBaseSecondsInitialValue);
            config.reportSettingAsGauge(retryBackoffBaseSeconds, "retryBackoffBaseSeconds");
            /*
          default value for pushMemoryBufferLimit is 16 * pushFlushMaxPoints, but no more than 25% of available heap
          memory. 25% is chosen heuristically as a safe number for scenarios with limited system resources (4 CPU cores
          or less, heap size less than 4GB) to prevent OOM. this is a conservative estimate, budgeting 200 characters
          (400 bytes) per per point line. Also, it shouldn't be less than 1 batch size (pushFlushMaxPoints).
         */
            int listeningPorts = Iterables.size(Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts));
            long calculatedMemoryBufferLimit = Math.max(Math.min(16 * pushFlushMaxPoints.get(), Runtime.getRuntime().maxMemory() / (listeningPorts > 0 ? listeningPorts : 1) / 4 / flushThreads / 400), pushFlushMaxPoints.get());
            logger.fine("Calculated pushMemoryBufferLimit: " + calculatedMemoryBufferLimit);
            pushMemoryBufferLimit.set(Integer.parseInt(config.getRawProperty("pushMemoryBufferLimit", String.valueOf(pushMemoryBufferLimit.get())).trim()));
            config.reportSettingAsGauge(pushMemoryBufferLimit, "pushMemoryBufferLimit");
            logger.fine("Configured pushMemoryBufferLimit: " + pushMemoryBufferLimit);
            logger.warning("Loaded configuration file " + pushConfigFile);
        } catch (Throwable exception) {
            logger.severe("Could not load configuration file " + pushConfigFile);
            throw exception;
        }
        // Compatibility with deprecated fields
        if (whitelistRegex == null && graphiteWhitelistRegex != null) {
            whitelistRegex = graphiteWhitelistRegex;
        }
        if (blacklistRegex == null && graphiteBlacklistRegex != null) {
            blacklistRegex = graphiteBlacklistRegex;
        }
        initPreprocessors();
        if (!persistMessages) {
            persistMessagesCompression = false;
        }
        if (pushRateLimit > 0) {
            pushRateLimiter = RecyclableRateLimiter.create(pushRateLimit, pushRateLimitMaxBurstSeconds);
        }
        pushMemoryBufferLimit.set(Math.max(pushMemoryBufferLimit.get(), pushFlushMaxPoints.get()));
        PostPushDataTimedTask.setPointsPerBatch(pushFlushMaxPoints);
        PostPushDataTimedTask.setMemoryBufferLimit(pushMemoryBufferLimit);
        QueuedAgentService.setSplitBatchSize(pushFlushMaxPoints);
        retryBackoffBaseSeconds.set(Math.max(Math.min(retryBackoffBaseSeconds.get(), MAX_RETRY_BACKOFF_BASE_SECONDS), 1.0));
        QueuedAgentService.setRetryBackoffBaseSeconds(retryBackoffBaseSeconds);
        // for backwards compatibility - if pushLogLevel is defined in the config file, change log level programmatically
        Level level = null;
        switch(pushLogLevel) {
            case "NONE":
                level = Level.WARNING;
                break;
            case "SUMMARY":
                level = Level.INFO;
                break;
            case "DETAILED":
                level = Level.FINE;
                break;
        }
        if (level != null) {
            Logger.getLogger("agent").setLevel(level);
            Logger.getLogger(PostPushDataTimedTask.class.getCanonicalName()).setLevel(level);
            Logger.getLogger(QueuedAgentService.class.getCanonicalName()).setLevel(level);
        }
    }
}