use of com.wavefront.common.NamedThreadFactory in project java by wavefrontHQ.
the class QueueingFactoryImpl method getQueueController.
@SuppressWarnings("unchecked")
@Override
public <T extends DataSubmissionTask<T>> QueueController<T> getQueueController(@Nonnull HandlerKey handlerKey, int numThreads) {
ScheduledExecutorService executor = executors.computeIfAbsent(handlerKey, x -> Executors.newScheduledThreadPool(numThreads, new NamedThreadFactory("queueProcessor-" + handlerKey.getEntityType() + "-" + handlerKey.getHandle())));
List<QueueProcessor<T>> queueProcessors = IntStream.range(0, numThreads).mapToObj(i -> (QueueProcessor<T>) getQueueProcessor(handlerKey, executor, i)).collect(Collectors.toList());
return (QueueController<T>) queueControllers.computeIfAbsent(handlerKey, x -> new QueueController<>(handlerKey, queueProcessors, backlogSize -> entityPropsFactory.get(handlerKey.getEntityType()).reportBacklogSize(handlerKey.getHandle(), backlogSize)));
}
use of com.wavefront.common.NamedThreadFactory in project java by wavefrontHQ.
the class PushAgent method startListeners.
@Override
protected void startListeners() throws Exception {
blockedPointsLogger = Logger.getLogger(proxyConfig.getBlockedPointsLoggerName());
blockedHistogramsLogger = Logger.getLogger(proxyConfig.getBlockedHistogramsLoggerName());
blockedSpansLogger = Logger.getLogger(proxyConfig.getBlockedSpansLoggerName());
if (proxyConfig.getSoLingerTime() >= 0) {
childChannelOptions.put(ChannelOption.SO_LINGER, proxyConfig.getSoLingerTime());
}
hostnameResolver = new CachingHostnameLookupResolver(proxyConfig.isDisableRdnsLookup(), ExpectedAgentMetric.RDNS_CACHE_SIZE.metricName);
if (proxyConfig.isSqsQueueBuffer()) {
taskQueueFactory = new SQSQueueFactoryImpl(proxyConfig.getSqsQueueNameTemplate(), proxyConfig.getSqsQueueRegion(), proxyConfig.getSqsQueueIdentifier(), proxyConfig.isPurgeBuffer());
} else {
taskQueueFactory = new TaskQueueFactoryImpl(proxyConfig.getBufferFile(), proxyConfig.isPurgeBuffer(), proxyConfig.isDisableBufferSharding(), proxyConfig.getBufferShardSize());
}
remoteHostAnnotator = new SharedGraphiteHostAnnotator(proxyConfig.getCustomSourceTags(), hostnameResolver);
queueingFactory = new QueueingFactoryImpl(apiContainer, agentId, taskQueueFactory, entityProps);
senderTaskFactory = new SenderTaskFactoryImpl(apiContainer, agentId, taskQueueFactory, queueingFactory, entityProps);
if (proxyConfig.isHistogramPassthroughRecompression()) {
histogramRecompressor = new HistogramRecompressor(() -> entityProps.getGlobalProperties().getHistogramStorageAccuracy());
}
handlerFactory = new ReportableEntityHandlerFactoryImpl(senderTaskFactory, proxyConfig.getPushBlockedSamples(), validationConfiguration, blockedPointsLogger, blockedHistogramsLogger, blockedSpansLogger, histogramRecompressor, entityProps);
if (proxyConfig.isTrafficShaping()) {
new TrafficShapingRateLimitAdjuster(entityProps, proxyConfig.getTrafficShapingWindowSeconds(), proxyConfig.getTrafficShapingHeadroom()).start();
}
healthCheckManager = new HealthCheckManagerImpl(proxyConfig);
tokenAuthenticator = configureTokenAuthenticator();
shutdownTasks.add(() -> senderTaskFactory.shutdown());
shutdownTasks.add(() -> senderTaskFactory.drainBuffersToQueue(null));
// sampler for spans
rateSampler.setSamplingRate(entityProps.getGlobalProperties().getTraceSamplingRate());
Sampler durationSampler = SpanSamplerUtils.getDurationSampler(proxyConfig.getTraceSamplingDuration());
List<Sampler> samplers = SpanSamplerUtils.fromSamplers(rateSampler, durationSampler);
SpanSampler spanSampler = new SpanSampler(new CompositeSampler(samplers), () -> entityProps.getGlobalProperties().getActiveSpanSamplingPolicies());
if (proxyConfig.getAdminApiListenerPort() > 0) {
startAdminListener(proxyConfig.getAdminApiListenerPort());
}
csvToList(proxyConfig.getHttpHealthCheckPorts()).forEach(strPort -> startHealthCheckListener(Integer.parseInt(strPort)));
csvToList(proxyConfig.getPushListenerPorts()).forEach(strPort -> {
startGraphiteListener(strPort, handlerFactory, remoteHostAnnotator, spanSampler);
logger.info("listening on port: " + strPort + " for Wavefront metrics");
});
csvToList(proxyConfig.getDeltaCountersAggregationListenerPorts()).forEach(strPort -> {
startDeltaCounterListener(strPort, remoteHostAnnotator, senderTaskFactory, spanSampler);
logger.info("listening on port: " + strPort + " for Wavefront delta counter metrics");
});
{
// Histogram bootstrap.
List<String> histMinPorts = csvToList(proxyConfig.getHistogramMinuteListenerPorts());
List<String> histHourPorts = csvToList(proxyConfig.getHistogramHourListenerPorts());
List<String> histDayPorts = csvToList(proxyConfig.getHistogramDayListenerPorts());
List<String> histDistPorts = csvToList(proxyConfig.getHistogramDistListenerPorts());
int activeHistogramAggregationTypes = (histDayPorts.size() > 0 ? 1 : 0) + (histHourPorts.size() > 0 ? 1 : 0) + (histMinPorts.size() > 0 ? 1 : 0) + (histDistPorts.size() > 0 ? 1 : 0);
if (activeHistogramAggregationTypes > 0) {
/*Histograms enabled*/
histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service"));
histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-flush"));
managedExecutors.add(histogramExecutor);
managedExecutors.add(histogramFlushExecutor);
File baseDirectory = new File(proxyConfig.getHistogramStateDirectory());
// Central dispatch
ReportableEntityHandler<ReportPoint, String> pointHandler = handlerFactory.getHandler(HandlerKey.of(ReportableEntityType.HISTOGRAM, "histogram_ports"));
startHistogramListeners(histMinPorts, pointHandler, remoteHostAnnotator, Granularity.MINUTE, proxyConfig.getHistogramMinuteFlushSecs(), proxyConfig.isHistogramMinuteMemoryCache(), baseDirectory, proxyConfig.getHistogramMinuteAccumulatorSize(), proxyConfig.getHistogramMinuteAvgKeyBytes(), proxyConfig.getHistogramMinuteAvgDigestBytes(), proxyConfig.getHistogramMinuteCompression(), proxyConfig.isHistogramMinuteAccumulatorPersisted(), spanSampler);
startHistogramListeners(histHourPorts, pointHandler, remoteHostAnnotator, Granularity.HOUR, proxyConfig.getHistogramHourFlushSecs(), proxyConfig.isHistogramHourMemoryCache(), baseDirectory, proxyConfig.getHistogramHourAccumulatorSize(), proxyConfig.getHistogramHourAvgKeyBytes(), proxyConfig.getHistogramHourAvgDigestBytes(), proxyConfig.getHistogramHourCompression(), proxyConfig.isHistogramHourAccumulatorPersisted(), spanSampler);
startHistogramListeners(histDayPorts, pointHandler, remoteHostAnnotator, Granularity.DAY, proxyConfig.getHistogramDayFlushSecs(), proxyConfig.isHistogramDayMemoryCache(), baseDirectory, proxyConfig.getHistogramDayAccumulatorSize(), proxyConfig.getHistogramDayAvgKeyBytes(), proxyConfig.getHistogramDayAvgDigestBytes(), proxyConfig.getHistogramDayCompression(), proxyConfig.isHistogramDayAccumulatorPersisted(), spanSampler);
startHistogramListeners(histDistPorts, pointHandler, remoteHostAnnotator, null, proxyConfig.getHistogramDistFlushSecs(), proxyConfig.isHistogramDistMemoryCache(), baseDirectory, proxyConfig.getHistogramDistAccumulatorSize(), proxyConfig.getHistogramDistAvgKeyBytes(), proxyConfig.getHistogramDistAvgDigestBytes(), proxyConfig.getHistogramDistCompression(), proxyConfig.isHistogramDistAccumulatorPersisted(), spanSampler);
}
}
if (StringUtils.isNotBlank(proxyConfig.getGraphitePorts()) || StringUtils.isNotBlank(proxyConfig.getPicklePorts())) {
if (tokenAuthenticator.authRequired()) {
logger.warning("Graphite mode is not compatible with HTTP authentication, ignoring");
} else {
Preconditions.checkNotNull(proxyConfig.getGraphiteFormat(), "graphiteFormat must be supplied to enable graphite support");
Preconditions.checkNotNull(proxyConfig.getGraphiteDelimiters(), "graphiteDelimiters must be supplied to enable graphite support");
GraphiteFormatter graphiteFormatter = new GraphiteFormatter(proxyConfig.getGraphiteFormat(), proxyConfig.getGraphiteDelimiters(), proxyConfig.getGraphiteFieldsToRemove());
csvToList(proxyConfig.getGraphitePorts()).forEach(strPort -> {
preprocessors.getSystemPreprocessor(strPort).forPointLine().addTransformer(0, graphiteFormatter);
startGraphiteListener(strPort, handlerFactory, null, spanSampler);
logger.info("listening on port: " + strPort + " for graphite metrics");
});
csvToList(proxyConfig.getPicklePorts()).forEach(strPort -> startPickleListener(strPort, handlerFactory, graphiteFormatter));
}
}
csvToList(proxyConfig.getOpentsdbPorts()).forEach(strPort -> startOpenTsdbListener(strPort, handlerFactory));
if (proxyConfig.getDataDogJsonPorts() != null) {
HttpClient httpClient = HttpClientBuilder.create().useSystemProperties().setUserAgent(proxyConfig.getHttpUserAgent()).setConnectionTimeToLive(1, TimeUnit.MINUTES).setMaxConnPerRoute(100).setMaxConnTotal(100).setRetryHandler(new DefaultHttpRequestRetryHandler(proxyConfig.getHttpAutoRetries(), true)).setDefaultRequestConfig(RequestConfig.custom().setContentCompressionEnabled(true).setRedirectsEnabled(true).setConnectTimeout(proxyConfig.getHttpConnectTimeout()).setConnectionRequestTimeout(proxyConfig.getHttpConnectTimeout()).setSocketTimeout(proxyConfig.getHttpRequestTimeout()).build()).build();
csvToList(proxyConfig.getDataDogJsonPorts()).forEach(strPort -> startDataDogListener(strPort, handlerFactory, httpClient));
}
csvToList(proxyConfig.getTraceListenerPorts()).forEach(strPort -> startTraceListener(strPort, handlerFactory, spanSampler));
csvToList(proxyConfig.getCustomTracingListenerPorts()).forEach(strPort -> startCustomTracingListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler));
csvToList(proxyConfig.getTraceJaegerListenerPorts()).forEach(strPort -> {
PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
startTraceJaegerListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
});
csvToList(proxyConfig.getTraceJaegerGrpcListenerPorts()).forEach(strPort -> {
PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
startTraceJaegerGrpcListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
});
csvToList(proxyConfig.getTraceJaegerHttpListenerPorts()).forEach(strPort -> {
PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
startTraceJaegerHttpListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
});
csvToList(proxyConfig.getTraceZipkinListenerPorts()).forEach(strPort -> {
PreprocessorRuleMetrics ruleMetrics = new PreprocessorRuleMetrics(Metrics.newCounter(new TaggedMetricName("point.spanSanitize", "count", "port", strPort)), null, null);
preprocessors.getSystemPreprocessor(strPort).forSpan().addTransformer(new SpanSanitizeTransformer(ruleMetrics));
startTraceZipkinListener(strPort, handlerFactory, new InternalProxyWavefrontClient(handlerFactory, strPort), spanSampler);
});
csvToList(proxyConfig.getPushRelayListenerPorts()).forEach(strPort -> startRelayListener(strPort, handlerFactory, remoteHostAnnotator));
csvToList(proxyConfig.getJsonListenerPorts()).forEach(strPort -> startJsonListener(strPort, handlerFactory));
csvToList(proxyConfig.getWriteHttpJsonListenerPorts()).forEach(strPort -> startWriteHttpJsonListener(strPort, handlerFactory));
// Logs ingestion.
if (proxyConfig.getFilebeatPort() > 0 || proxyConfig.getRawLogsPort() > 0) {
if (loadLogsIngestionConfig() != null) {
logger.info("Initializing logs ingestion");
try {
final LogsIngester logsIngester = new LogsIngester(handlerFactory, this::loadLogsIngestionConfig, proxyConfig.getPrefix());
logsIngester.start();
if (proxyConfig.getFilebeatPort() > 0) {
startLogsIngestionListener(proxyConfig.getFilebeatPort(), logsIngester);
}
if (proxyConfig.getRawLogsPort() > 0) {
startRawLogsIngestionListener(proxyConfig.getRawLogsPort(), logsIngester);
}
} catch (ConfigurationException e) {
logger.log(Level.SEVERE, "Cannot start logsIngestion", e);
}
} else {
logger.warning("Cannot start logsIngestion: invalid configuration or no config specified");
}
}
setupMemoryGuard();
}
use of com.wavefront.common.NamedThreadFactory in project java by wavefrontHQ.
the class SenderTaskFactoryImpl method createSenderTasks.
@SuppressWarnings("unchecked")
public Collection<SenderTask<?>> createSenderTasks(@Nonnull HandlerKey handlerKey) {
ReportableEntityType entityType = handlerKey.getEntityType();
int numThreads = entityPropsFactory.get(entityType).getFlushThreads();
List<SenderTask<?>> toReturn = new ArrayList<>(numThreads);
TaskSizeEstimator taskSizeEstimator = new TaskSizeEstimator(handlerKey.getHandle());
taskSizeEstimators.put(handlerKey, taskSizeEstimator);
ScheduledExecutorService scheduler = executors.computeIfAbsent(handlerKey, x -> Executors.newScheduledThreadPool(numThreads, new NamedThreadFactory("submitter-" + handlerKey.getEntityType() + "-" + handlerKey.getHandle())));
for (int threadNo = 0; threadNo < numThreads; threadNo++) {
SenderTask<?> senderTask;
switch(entityType) {
case POINT:
case DELTA_COUNTER:
senderTask = new LineDelimitedSenderTask(handlerKey, PUSH_FORMAT_WAVEFRONT, apiContainer.getProxyV2API(), proxyId, entityPropsFactory.get(entityType), scheduler, threadNo, taskSizeEstimator, taskQueueFactory.getTaskQueue(handlerKey, threadNo));
break;
case HISTOGRAM:
senderTask = new LineDelimitedSenderTask(handlerKey, PUSH_FORMAT_HISTOGRAM, apiContainer.getProxyV2API(), proxyId, entityPropsFactory.get(entityType), scheduler, threadNo, taskSizeEstimator, taskQueueFactory.getTaskQueue(handlerKey, threadNo));
break;
case SOURCE_TAG:
senderTask = new SourceTagSenderTask(handlerKey, apiContainer.getSourceTagAPI(), threadNo, entityPropsFactory.get(entityType), scheduler, taskQueueFactory.getTaskQueue(handlerKey, threadNo));
break;
case TRACE:
senderTask = new LineDelimitedSenderTask(handlerKey, PUSH_FORMAT_TRACING, apiContainer.getProxyV2API(), proxyId, entityPropsFactory.get(entityType), scheduler, threadNo, taskSizeEstimator, taskQueueFactory.getTaskQueue(handlerKey, threadNo));
break;
case TRACE_SPAN_LOGS:
senderTask = new LineDelimitedSenderTask(handlerKey, PUSH_FORMAT_TRACING_SPAN_LOGS, apiContainer.getProxyV2API(), proxyId, entityPropsFactory.get(entityType), scheduler, threadNo, taskSizeEstimator, taskQueueFactory.getTaskQueue(handlerKey, threadNo));
break;
case EVENT:
senderTask = new EventSenderTask(handlerKey, apiContainer.getEventAPI(), proxyId, threadNo, entityPropsFactory.get(entityType), scheduler, taskQueueFactory.getTaskQueue(handlerKey, threadNo));
break;
default:
throw new IllegalArgumentException("Unexpected entity type " + handlerKey.getEntityType().name() + " for " + handlerKey.getHandle());
}
toReturn.add(senderTask);
senderTask.start();
}
if (queueingFactory != null) {
QueueController<?> controller = queueingFactory.getQueueController(handlerKey, numThreads);
managedServices.put(handlerKey, controller);
controller.start();
}
managedTasks.put(handlerKey, toReturn);
entityTypes.computeIfAbsent(handlerKey.getHandle(), x -> new ArrayList<>()).add(handlerKey.getEntityType());
return toReturn;
}
Aggregations