use of com.wavefront.agent.data.EntityProperties.DEFAULT_BATCH_SIZE_HISTOGRAMS in project java by wavefrontHQ.
the class ProxyConfig method verifyAndInit.
@Override
public void verifyAndInit() {
if (unparsed_params != null) {
logger.info("Unparsed arguments: " + Joiner.on(", ").join(unparsed_params));
}
ReportableConfig config;
// If they've specified a push configuration file, override the command line values
try {
if (pushConfigFile != null) {
config = new ReportableConfig(pushConfigFile);
} else {
// dummy config
config = new ReportableConfig();
}
prefix = Strings.emptyToNull(config.getString("prefix", prefix));
// don't track token in proxy config metrics
token = ObjectUtils.firstNonNull(config.getRawProperty("token", token), "undefined").trim();
server = config.getString("server", server);
hostname = config.getString("hostname", hostname);
idFile = config.getString("idFile", idFile);
pushRateLimit = config.getInteger("pushRateLimit", pushRateLimit);
pushRateLimitHistograms = config.getInteger("pushRateLimitHistograms", pushRateLimitHistograms);
pushRateLimitSourceTags = config.getDouble("pushRateLimitSourceTags", pushRateLimitSourceTags);
pushRateLimitSpans = config.getInteger("pushRateLimitSpans", pushRateLimitSpans);
pushRateLimitSpanLogs = config.getInteger("pushRateLimitSpanLogs", pushRateLimitSpanLogs);
pushRateLimitEvents = config.getDouble("pushRateLimitEvents", pushRateLimitEvents);
pushRateLimitMaxBurstSeconds = config.getInteger("pushRateLimitMaxBurstSeconds", pushRateLimitMaxBurstSeconds);
pushBlockedSamples = config.getInteger("pushBlockedSamples", pushBlockedSamples);
blockedPointsLoggerName = config.getString("blockedPointsLoggerName", blockedPointsLoggerName);
blockedHistogramsLoggerName = config.getString("blockedHistogramsLoggerName", blockedHistogramsLoggerName);
blockedSpansLoggerName = config.getString("blockedSpansLoggerName", blockedSpansLoggerName);
pushListenerPorts = config.getString("pushListenerPorts", pushListenerPorts);
pushListenerMaxReceivedLength = config.getInteger("pushListenerMaxReceivedLength", pushListenerMaxReceivedLength);
pushListenerHttpBufferSize = config.getInteger("pushListenerHttpBufferSize", pushListenerHttpBufferSize);
traceListenerMaxReceivedLength = config.getInteger("traceListenerMaxReceivedLength", traceListenerMaxReceivedLength);
traceListenerHttpBufferSize = config.getInteger("traceListenerHttpBufferSize", traceListenerHttpBufferSize);
listenerIdleConnectionTimeout = config.getInteger("listenerIdleConnectionTimeout", listenerIdleConnectionTimeout);
memGuardFlushThreshold = config.getInteger("memGuardFlushThreshold", memGuardFlushThreshold);
// Histogram: global settings
histogramPassthroughRecompression = config.getBoolean("histogramPassthroughRecompression", histogramPassthroughRecompression);
histogramStateDirectory = config.getString("histogramStateDirectory", histogramStateDirectory);
histogramAccumulatorResolveInterval = config.getLong("histogramAccumulatorResolveInterval", histogramAccumulatorResolveInterval);
histogramAccumulatorFlushInterval = config.getLong("histogramAccumulatorFlushInterval", histogramAccumulatorFlushInterval);
histogramAccumulatorFlushMaxBatchSize = config.getInteger("histogramAccumulatorFlushMaxBatchSize", histogramAccumulatorFlushMaxBatchSize);
histogramMaxReceivedLength = config.getInteger("histogramMaxReceivedLength", histogramMaxReceivedLength);
histogramHttpBufferSize = config.getInteger("histogramHttpBufferSize", histogramHttpBufferSize);
deltaCountersAggregationListenerPorts = config.getString("deltaCountersAggregationListenerPorts", deltaCountersAggregationListenerPorts);
deltaCountersAggregationIntervalSeconds = config.getLong("deltaCountersAggregationIntervalSeconds", deltaCountersAggregationIntervalSeconds);
customTracingListenerPorts = config.getString("customTracingListenerPorts", customTracingListenerPorts);
// Histogram: deprecated settings - fall back for backwards compatibility
if (config.isDefined("avgHistogramKeyBytes")) {
histogramMinuteAvgKeyBytes = histogramHourAvgKeyBytes = histogramDayAvgKeyBytes = histogramDistAvgKeyBytes = config.getInteger("avgHistogramKeyBytes", 150);
}
if (config.isDefined("avgHistogramDigestBytes")) {
histogramMinuteAvgDigestBytes = histogramHourAvgDigestBytes = histogramDayAvgDigestBytes = histogramDistAvgDigestBytes = config.getInteger("avgHistogramDigestBytes", 500);
}
if (config.isDefined("histogramAccumulatorSize")) {
histogramMinuteAccumulatorSize = histogramHourAccumulatorSize = histogramDayAccumulatorSize = histogramDistAccumulatorSize = config.getLong("histogramAccumulatorSize", 100000);
}
if (config.isDefined("histogramCompression")) {
histogramMinuteCompression = histogramHourCompression = histogramDayCompression = histogramDistCompression = config.getNumber("histogramCompression", null, 20, 1000).shortValue();
}
if (config.isDefined("persistAccumulator")) {
histogramMinuteAccumulatorPersisted = histogramHourAccumulatorPersisted = histogramDayAccumulatorPersisted = histogramDistAccumulatorPersisted = config.getBoolean("persistAccumulator", false);
}
// Histogram: minute accumulator settings
histogramMinuteListenerPorts = config.getString("histogramMinuteListenerPorts", histogramMinuteListenerPorts);
histogramMinuteFlushSecs = config.getInteger("histogramMinuteFlushSecs", histogramMinuteFlushSecs);
histogramMinuteCompression = config.getNumber("histogramMinuteCompression", histogramMinuteCompression, 20, 1000).shortValue();
histogramMinuteAvgKeyBytes = config.getInteger("histogramMinuteAvgKeyBytes", histogramMinuteAvgKeyBytes);
histogramMinuteAvgDigestBytes = 32 + histogramMinuteCompression * 7;
histogramMinuteAvgDigestBytes = config.getInteger("histogramMinuteAvgDigestBytes", histogramMinuteAvgDigestBytes);
histogramMinuteAccumulatorSize = config.getLong("histogramMinuteAccumulatorSize", histogramMinuteAccumulatorSize);
histogramMinuteAccumulatorPersisted = config.getBoolean("histogramMinuteAccumulatorPersisted", histogramMinuteAccumulatorPersisted);
histogramMinuteMemoryCache = config.getBoolean("histogramMinuteMemoryCache", histogramMinuteMemoryCache);
// Histogram: hour accumulator settings
histogramHourListenerPorts = config.getString("histogramHourListenerPorts", histogramHourListenerPorts);
histogramHourFlushSecs = config.getInteger("histogramHourFlushSecs", histogramHourFlushSecs);
histogramHourCompression = config.getNumber("histogramHourCompression", histogramHourCompression, 20, 1000).shortValue();
histogramHourAvgKeyBytes = config.getInteger("histogramHourAvgKeyBytes", histogramHourAvgKeyBytes);
histogramHourAvgDigestBytes = 32 + histogramHourCompression * 7;
histogramHourAvgDigestBytes = config.getInteger("histogramHourAvgDigestBytes", histogramHourAvgDigestBytes);
histogramHourAccumulatorSize = config.getLong("histogramHourAccumulatorSize", histogramHourAccumulatorSize);
histogramHourAccumulatorPersisted = config.getBoolean("histogramHourAccumulatorPersisted", histogramHourAccumulatorPersisted);
histogramHourMemoryCache = config.getBoolean("histogramHourMemoryCache", histogramHourMemoryCache);
// Histogram: day accumulator settings
histogramDayListenerPorts = config.getString("histogramDayListenerPorts", histogramDayListenerPorts);
histogramDayFlushSecs = config.getInteger("histogramDayFlushSecs", histogramDayFlushSecs);
histogramDayCompression = config.getNumber("histogramDayCompression", histogramDayCompression, 20, 1000).shortValue();
histogramDayAvgKeyBytes = config.getInteger("histogramDayAvgKeyBytes", histogramDayAvgKeyBytes);
histogramDayAvgDigestBytes = 32 + histogramDayCompression * 7;
histogramDayAvgDigestBytes = config.getInteger("histogramDayAvgDigestBytes", histogramDayAvgDigestBytes);
histogramDayAccumulatorSize = config.getLong("histogramDayAccumulatorSize", histogramDayAccumulatorSize);
histogramDayAccumulatorPersisted = config.getBoolean("histogramDayAccumulatorPersisted", histogramDayAccumulatorPersisted);
histogramDayMemoryCache = config.getBoolean("histogramDayMemoryCache", histogramDayMemoryCache);
// Histogram: dist accumulator settings
histogramDistListenerPorts = config.getString("histogramDistListenerPorts", histogramDistListenerPorts);
histogramDistFlushSecs = config.getInteger("histogramDistFlushSecs", histogramDistFlushSecs);
histogramDistCompression = config.getNumber("histogramDistCompression", histogramDistCompression, 20, 1000).shortValue();
histogramDistAvgKeyBytes = config.getInteger("histogramDistAvgKeyBytes", histogramDistAvgKeyBytes);
histogramDistAvgDigestBytes = 32 + histogramDistCompression * 7;
histogramDistAvgDigestBytes = config.getInteger("histogramDistAvgDigestBytes", histogramDistAvgDigestBytes);
histogramDistAccumulatorSize = config.getLong("histogramDistAccumulatorSize", histogramDistAccumulatorSize);
histogramDistAccumulatorPersisted = config.getBoolean("histogramDistAccumulatorPersisted", histogramDistAccumulatorPersisted);
histogramDistMemoryCache = config.getBoolean("histogramDistMemoryCache", histogramDistMemoryCache);
exportQueuePorts = config.getString("exportQueuePorts", exportQueuePorts);
exportQueueOutputFile = config.getString("exportQueueOutputFile", exportQueueOutputFile);
exportQueueRetainData = config.getBoolean("exportQueueRetainData", exportQueueRetainData);
useNoopSender = config.getBoolean("useNoopSender", useNoopSender);
flushThreads = config.getInteger("flushThreads", flushThreads);
flushThreadsEvents = config.getInteger("flushThreadsEvents", flushThreadsEvents);
flushThreadsSourceTags = config.getInteger("flushThreadsSourceTags", flushThreadsSourceTags);
jsonListenerPorts = config.getString("jsonListenerPorts", jsonListenerPorts);
writeHttpJsonListenerPorts = config.getString("writeHttpJsonListenerPorts", writeHttpJsonListenerPorts);
dataDogJsonPorts = config.getString("dataDogJsonPorts", dataDogJsonPorts);
dataDogRequestRelayTarget = config.getString("dataDogRequestRelayTarget", dataDogRequestRelayTarget);
dataDogRequestRelayAsyncThreads = config.getInteger("dataDogRequestRelayAsyncThreads", dataDogRequestRelayAsyncThreads);
dataDogRequestRelaySyncMode = config.getBoolean("dataDogRequestRelaySyncMode", dataDogRequestRelaySyncMode);
dataDogProcessSystemMetrics = config.getBoolean("dataDogProcessSystemMetrics", dataDogProcessSystemMetrics);
dataDogProcessServiceChecks = config.getBoolean("dataDogProcessServiceChecks", dataDogProcessServiceChecks);
graphitePorts = config.getString("graphitePorts", graphitePorts);
graphiteFormat = config.getString("graphiteFormat", graphiteFormat);
graphiteFieldsToRemove = config.getString("graphiteFieldsToRemove", graphiteFieldsToRemove);
graphiteDelimiters = config.getString("graphiteDelimiters", graphiteDelimiters);
allowRegex = config.getString("allowRegex", config.getString("whitelistRegex", allowRegex));
blockRegex = config.getString("blockRegex", config.getString("blacklistRegex", blockRegex));
opentsdbPorts = config.getString("opentsdbPorts", opentsdbPorts);
opentsdbAllowRegex = config.getString("opentsdbAllowRegex", config.getString("opentsdbWhitelistRegex", opentsdbAllowRegex));
opentsdbBlockRegex = config.getString("opentsdbBlockRegex", config.getString("opentsdbBlacklistRegex", opentsdbBlockRegex));
proxyHost = config.getString("proxyHost", proxyHost);
proxyPort = config.getInteger("proxyPort", proxyPort);
proxyPassword = config.getString("proxyPassword", proxyPassword, s -> "<removed>");
proxyUser = config.getString("proxyUser", proxyUser);
httpUserAgent = config.getString("httpUserAgent", httpUserAgent);
httpConnectTimeout = config.getInteger("httpConnectTimeout", httpConnectTimeout);
httpRequestTimeout = config.getInteger("httpRequestTimeout", httpRequestTimeout);
httpMaxConnTotal = Math.min(200, config.getInteger("httpMaxConnTotal", httpMaxConnTotal));
httpMaxConnPerRoute = Math.min(100, config.getInteger("httpMaxConnPerRoute", httpMaxConnPerRoute));
httpAutoRetries = config.getInteger("httpAutoRetries", httpAutoRetries);
gzipCompression = config.getBoolean("gzipCompression", gzipCompression);
gzipCompressionLevel = config.getNumber("gzipCompressionLevel", gzipCompressionLevel, 1, 9).intValue();
soLingerTime = config.getInteger("soLingerTime", soLingerTime);
splitPushWhenRateLimited = config.getBoolean("splitPushWhenRateLimited", splitPushWhenRateLimited);
customSourceTags = config.getString("customSourceTags", customSourceTags);
agentMetricsPointTags = config.getString("agentMetricsPointTags", agentMetricsPointTags);
ephemeral = config.getBoolean("ephemeral", ephemeral);
disableRdnsLookup = config.getBoolean("disableRdnsLookup", disableRdnsLookup);
picklePorts = config.getString("picklePorts", picklePorts);
traceListenerPorts = config.getString("traceListenerPorts", traceListenerPorts);
traceJaegerListenerPorts = config.getString("traceJaegerListenerPorts", traceJaegerListenerPorts);
traceJaegerHttpListenerPorts = config.getString("traceJaegerHttpListenerPorts", traceJaegerHttpListenerPorts);
traceJaegerGrpcListenerPorts = config.getString("traceJaegerGrpcListenerPorts", traceJaegerGrpcListenerPorts);
traceJaegerApplicationName = config.getString("traceJaegerApplicationName", traceJaegerApplicationName);
traceZipkinListenerPorts = config.getString("traceZipkinListenerPorts", traceZipkinListenerPorts);
traceZipkinApplicationName = config.getString("traceZipkinApplicationName", traceZipkinApplicationName);
customTracingListenerPorts = config.getString("customTracingListenerPorts", customTracingListenerPorts);
customTracingApplicationName = config.getString("customTracingApplicationName", customTracingApplicationName);
customTracingServiceName = config.getString("customTracingServiceName", customTracingServiceName);
traceSamplingRate = config.getDouble("traceSamplingRate", traceSamplingRate);
traceSamplingDuration = config.getInteger("traceSamplingDuration", traceSamplingDuration);
traceDerivedCustomTagKeys = config.getString("traceDerivedCustomTagKeys", traceDerivedCustomTagKeys);
backendSpanHeadSamplingPercentIgnored = config.getBoolean("backendSpanHeadSamplingPercentIgnored", backendSpanHeadSamplingPercentIgnored);
pushRelayListenerPorts = config.getString("pushRelayListenerPorts", pushRelayListenerPorts);
pushRelayHistogramAggregator = config.getBoolean("pushRelayHistogramAggregator", pushRelayHistogramAggregator);
pushRelayHistogramAggregatorAccumulatorSize = config.getLong("pushRelayHistogramAggregatorAccumulatorSize", pushRelayHistogramAggregatorAccumulatorSize);
pushRelayHistogramAggregatorFlushSecs = config.getInteger("pushRelayHistogramAggregatorFlushSecs", pushRelayHistogramAggregatorFlushSecs);
pushRelayHistogramAggregatorCompression = config.getNumber("pushRelayHistogramAggregatorCompression", pushRelayHistogramAggregatorCompression).shortValue();
bufferFile = config.getString("buffer", bufferFile);
bufferShardSize = config.getInteger("bufferShardSize", bufferShardSize);
disableBufferSharding = config.getBoolean("disableBufferSharding", disableBufferSharding);
taskQueueLevel = TaskQueueLevel.fromString(config.getString("taskQueueStrategy", taskQueueLevel.toString()));
purgeBuffer = config.getBoolean("purgeBuffer", purgeBuffer);
preprocessorConfigFile = config.getString("preprocessorConfigFile", preprocessorConfigFile);
dataBackfillCutoffHours = config.getInteger("dataBackfillCutoffHours", dataBackfillCutoffHours);
dataPrefillCutoffHours = config.getInteger("dataPrefillCutoffHours", dataPrefillCutoffHours);
filebeatPort = config.getInteger("filebeatPort", filebeatPort);
rawLogsPort = config.getInteger("rawLogsPort", rawLogsPort);
rawLogsMaxReceivedLength = config.getInteger("rawLogsMaxReceivedLength", rawLogsMaxReceivedLength);
rawLogsHttpBufferSize = config.getInteger("rawLogsHttpBufferSize", rawLogsHttpBufferSize);
logsIngestionConfigFile = config.getString("logsIngestionConfigFile", logsIngestionConfigFile);
sqsQueueBuffer = config.getBoolean("sqsBuffer", sqsQueueBuffer);
sqsQueueNameTemplate = config.getString("sqsQueueNameTemplate", sqsQueueNameTemplate);
sqsQueueRegion = config.getString("sqsQueueRegion", sqsQueueRegion);
sqsQueueIdentifier = config.getString("sqsQueueIdentifier", sqsQueueIdentifier);
// auth settings
authMethod = TokenValidationMethod.fromString(config.getString("authMethod", authMethod.toString()));
authTokenIntrospectionServiceUrl = config.getString("authTokenIntrospectionServiceUrl", authTokenIntrospectionServiceUrl);
authTokenIntrospectionAuthorizationHeader = config.getString("authTokenIntrospectionAuthorizationHeader", authTokenIntrospectionAuthorizationHeader);
authResponseRefreshInterval = config.getInteger("authResponseRefreshInterval", authResponseRefreshInterval);
authResponseMaxTtl = config.getInteger("authResponseMaxTtl", authResponseMaxTtl);
authStaticToken = config.getString("authStaticToken", authStaticToken);
// health check / admin API settings
adminApiListenerPort = config.getInteger("adminApiListenerPort", adminApiListenerPort);
adminApiRemoteIpAllowRegex = config.getString("adminApiRemoteIpWhitelistRegex", adminApiRemoteIpAllowRegex);
httpHealthCheckPorts = config.getString("httpHealthCheckPorts", httpHealthCheckPorts);
httpHealthCheckAllPorts = config.getBoolean("httpHealthCheckAllPorts", false);
httpHealthCheckPath = config.getString("httpHealthCheckPath", httpHealthCheckPath);
httpHealthCheckResponseContentType = config.getString("httpHealthCheckResponseContentType", httpHealthCheckResponseContentType);
httpHealthCheckPassStatusCode = config.getInteger("httpHealthCheckPassStatusCode", httpHealthCheckPassStatusCode);
httpHealthCheckPassResponseBody = config.getString("httpHealthCheckPassResponseBody", httpHealthCheckPassResponseBody);
httpHealthCheckFailStatusCode = config.getInteger("httpHealthCheckFailStatusCode", httpHealthCheckFailStatusCode);
httpHealthCheckFailResponseBody = config.getString("httpHealthCheckFailResponseBody", httpHealthCheckFailResponseBody);
// TLS configurations
privateCertPath = config.getString("privateCertPath", privateCertPath);
privateKeyPath = config.getString("privateKeyPath", privateKeyPath);
tlsPorts = config.getString("tlsPorts", tlsPorts);
// Traffic shaping config
trafficShaping = config.getBoolean("trafficShaping", trafficShaping);
trafficShapingWindowSeconds = config.getInteger("trafficShapingWindowSeconds", trafficShapingWindowSeconds);
trafficShapingHeadroom = config.getDouble("trafficShapingHeadroom", trafficShapingHeadroom);
// CORS configuration
corsEnabledPorts = config.getString("corsEnabledPorts", corsEnabledPorts);
corsOrigin = config.getString("corsOrigin", corsOrigin);
corsAllowNullOrigin = config.getBoolean("corsAllowNullOrigin", corsAllowNullOrigin);
// clamp values for pushFlushMaxPoints/etc between min split size
// (or 1 in case of source tags and events) and default batch size.
// also make sure it is never higher than the configured rate limit.
pushFlushMaxPoints = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxPoints", pushFlushMaxPoints), DEFAULT_BATCH_SIZE), (int) pushRateLimit), DEFAULT_MIN_SPLIT_BATCH_SIZE);
pushFlushMaxHistograms = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxHistograms", pushFlushMaxHistograms), DEFAULT_BATCH_SIZE_HISTOGRAMS), (int) pushRateLimitHistograms), DEFAULT_MIN_SPLIT_BATCH_SIZE);
pushFlushMaxSourceTags = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxSourceTags", pushFlushMaxSourceTags), DEFAULT_BATCH_SIZE_SOURCE_TAGS), (int) pushRateLimitSourceTags), 1);
pushFlushMaxSpans = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxSpans", pushFlushMaxSpans), DEFAULT_BATCH_SIZE_SPANS), (int) pushRateLimitSpans), DEFAULT_MIN_SPLIT_BATCH_SIZE);
pushFlushMaxSpanLogs = Math.max(Math.min(Math.min(config.getInteger("pushFlushMaxSpanLogs", pushFlushMaxSpanLogs), DEFAULT_BATCH_SIZE_SPAN_LOGS), (int) pushRateLimitSpanLogs), DEFAULT_MIN_SPLIT_BATCH_SIZE);
pushFlushMaxEvents = Math.min(Math.min(Math.max(config.getInteger("pushFlushMaxEvents", pushFlushMaxEvents), 1), DEFAULT_BATCH_SIZE_EVENTS), (int) (pushRateLimitEvents + 1));
/*
default value for pushMemoryBufferLimit is 16 * pushFlushMaxPoints, but no more than 25% of
available heap memory. 25% is chosen heuristically as a safe number for scenarios with
limited system resources (4 CPU cores or less, heap size less than 4GB) to prevent OOM.
this is a conservative estimate, budgeting 200 characters (400 bytes) per per point line.
Also, it shouldn't be less than 1 batch size (pushFlushMaxPoints).
*/
int listeningPorts = Iterables.size(Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts));
long calculatedMemoryBufferLimit = Math.max(Math.min(16 * pushFlushMaxPoints, Runtime.getRuntime().maxMemory() / Math.max(0, listeningPorts) / 4 / flushThreads / 400), pushFlushMaxPoints);
logger.fine("Calculated pushMemoryBufferLimit: " + calculatedMemoryBufferLimit);
pushMemoryBufferLimit = Math.max(config.getInteger("pushMemoryBufferLimit", pushMemoryBufferLimit), pushFlushMaxPoints);
logger.fine("Configured pushMemoryBufferLimit: " + pushMemoryBufferLimit);
pushFlushInterval = config.getInteger("pushFlushInterval", pushFlushInterval);
retryBackoffBaseSeconds = Math.max(Math.min(config.getDouble("retryBackoffBaseSeconds", retryBackoffBaseSeconds), MAX_RETRY_BACKOFF_BASE_SECONDS), 1.0);
} catch (Throwable exception) {
logger.severe("Could not load configuration file " + pushConfigFile);
throw new RuntimeException(exception.getMessage());
}
if (httpUserAgent == null) {
httpUserAgent = "Wavefront-Proxy/" + getBuildVersion();
}
if (pushConfigFile != null) {
logger.info("Loaded configuration file " + pushConfigFile);
}
}
Aggregations