use of com.wavefront.agent.histogram.tape.TapeDeck in project java by wavefrontHQ.
the class PushAgent method startListeners.
@Override
protected void startListeners() {
if (soLingerTime >= 0) {
childChannelOptions.put(ChannelOption.SO_LINGER, soLingerTime);
}
if (pushListenerPorts != null) {
Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(pushListenerPorts);
for (String strPort : ports) {
startGraphiteListener(strPort, false);
}
}
{
// Histogram bootstrap.
Iterator<String> histMinPorts = Strings.isNullOrEmpty(histogramMinuteListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramMinuteListenerPorts).iterator();
Iterator<String> histHourPorts = Strings.isNullOrEmpty(histogramHourListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramHourListenerPorts).iterator();
Iterator<String> histDayPorts = Strings.isNullOrEmpty(histogramDayListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramDayListenerPorts).iterator();
Iterator<String> histDistPorts = Strings.isNullOrEmpty(histogramDistListenerPorts) ? Collections.emptyIterator() : Splitter.on(",").omitEmptyStrings().trimResults().split(histogramDistListenerPorts).iterator();
int activeHistogramAggregationTypes = (histDayPorts.hasNext() ? 1 : 0) + (histHourPorts.hasNext() ? 1 : 0) + (histMinPorts.hasNext() ? 1 : 0) + (histDistPorts.hasNext() ? 1 : 0);
if (activeHistogramAggregationTypes > 0) {
/*Histograms enabled*/
histogramExecutor = Executors.newScheduledThreadPool(1 + activeHistogramAggregationTypes, new NamedThreadFactory("histogram-service"));
histogramFlushExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-flush"));
histogramScanExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors() / 2, new NamedThreadFactory("histogram-scan"));
managedExecutors.add(histogramExecutor);
managedExecutors.add(histogramFlushExecutor);
managedExecutors.add(histogramScanExecutor);
File baseDirectory = new File(histogramStateDirectory);
if (persistMessages || persistAccumulator) {
// Check directory
checkArgument(baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!");
checkArgument(baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!");
}
// Central dispatch
PointHandler histogramHandler = new PointHandlerImpl("histogram ports", pushValidationLevel, pushBlockedSamples, prefix, getFlushTasks(Constants.PUSH_FORMAT_HISTOGRAM, "histogram ports"));
// Input queue factory
TapeDeck<List<String>> accumulatorDeck = new TapeDeck<>(persistMessagesCompression ? TapeStringListConverter.getCompressionEnabledInstance() : TapeStringListConverter.getDefaultInstance(), persistMessages);
// Decoders
Decoder<String> sampleDecoder = new GraphiteDecoder("unknown", customSourceTags);
Decoder<String> distributionDecoder = new HistogramDecoder("unknown");
if (histMinPorts.hasNext()) {
startHistogramListeners(histMinPorts, sampleDecoder, histogramHandler, accumulatorDeck, "minute", histogramMinuteFlushSecs, histogramMinuteAccumulators, histogramMinuteMemoryCache, baseDirectory, histogramMinuteAccumulatorSize, histogramMinuteAvgKeyBytes, histogramMinuteAvgDigestBytes, histogramMinuteCompression);
}
if (histHourPorts.hasNext()) {
startHistogramListeners(histHourPorts, sampleDecoder, histogramHandler, accumulatorDeck, "hour", histogramHourFlushSecs, histogramHourAccumulators, histogramHourMemoryCache, baseDirectory, histogramHourAccumulatorSize, histogramHourAvgKeyBytes, histogramHourAvgDigestBytes, histogramHourCompression);
}
if (histDayPorts.hasNext()) {
startHistogramListeners(histDayPorts, sampleDecoder, histogramHandler, accumulatorDeck, "day", histogramDayFlushSecs, histogramDayAccumulators, histogramDayMemoryCache, baseDirectory, histogramDayAccumulatorSize, histogramDayAvgKeyBytes, histogramDayAvgDigestBytes, histogramDayCompression);
}
if (histDistPorts.hasNext()) {
startHistogramListeners(histDistPorts, distributionDecoder, histogramHandler, accumulatorDeck, "distribution", histogramDistFlushSecs, histogramDistAccumulators, histogramDistMemoryCache, baseDirectory, histogramDistAccumulatorSize, histogramDistAvgKeyBytes, histogramDistAvgDigestBytes, histogramDistCompression);
}
}
}
GraphiteFormatter graphiteFormatter = null;
if (graphitePorts != null || picklePorts != null) {
Preconditions.checkNotNull(graphiteFormat, "graphiteFormat must be supplied to enable graphite support");
Preconditions.checkNotNull(graphiteDelimiters, "graphiteDelimiters must be supplied to enable graphite support");
graphiteFormatter = new GraphiteFormatter(graphiteFormat, graphiteDelimiters, graphiteFieldsToRemove);
Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(graphitePorts);
for (String strPort : ports) {
preprocessors.forPort(strPort).forPointLine().addTransformer(0, graphiteFormatter);
startGraphiteListener(strPort, true);
logger.info("listening on port: " + strPort + " for graphite metrics");
}
}
if (opentsdbPorts != null) {
Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(opentsdbPorts);
for (String strPort : ports) {
startOpenTsdbListener(strPort);
logger.info("listening on port: " + strPort + " for OpenTSDB metrics");
}
}
if (picklePorts != null) {
Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(picklePorts);
for (String strPort : ports) {
startPickleListener(strPort, graphiteFormatter);
logger.info("listening on port: " + strPort + " for pickle protocol metrics");
}
}
if (httpJsonPorts != null) {
Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(httpJsonPorts);
for (String strPort : ports) {
preprocessors.forPort(strPort).forReportPoint().addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours));
startAsManagedThread(() -> {
activeListeners.inc();
try {
org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(Integer.parseInt(strPort));
server.setHandler(new JsonMetricsEndpoint(strPort, hostname, prefix, pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort), preprocessors.forPort(strPort)));
server.start();
server.join();
} catch (InterruptedException e) {
logger.warning("Http Json server interrupted.");
} catch (Exception e) {
if (e instanceof BindException) {
logger.severe("Unable to start listener - port " + String.valueOf(strPort) + " is already in use!");
}
} finally {
activeListeners.dec();
}
}, "listener-plaintext-json-" + strPort);
}
}
if (writeHttpJsonPorts != null) {
Iterable<String> ports = Splitter.on(",").omitEmptyStrings().trimResults().split(writeHttpJsonPorts);
for (String strPort : ports) {
preprocessors.forPort(strPort).forReportPoint().addFilter(new ReportPointTimestampInRangeFilter(dataBackfillCutoffHours, dataPrefillCutoffHours));
startAsManagedThread(() -> {
activeListeners.inc();
try {
org.eclipse.jetty.server.Server server = new org.eclipse.jetty.server.Server(Integer.parseInt(strPort));
server.setHandler(new WriteHttpJsonMetricsEndpoint(strPort, hostname, prefix, pushValidationLevel, pushBlockedSamples, getFlushTasks(strPort), preprocessors.forPort(strPort)));
server.start();
server.join();
} catch (InterruptedException e) {
logger.warning("WriteHttpJson server interrupted.");
} catch (Exception e) {
if (e instanceof BindException) {
logger.severe("Unable to start listener - port " + String.valueOf(strPort) + " is already in use!");
}
} finally {
activeListeners.dec();
}
}, "listener-plaintext-writehttpjson-" + strPort);
}
}
// Logs ingestion.
if (loadLogsIngestionConfig() != null) {
logger.info("Loading logs ingestion.");
try {
final LogsIngester logsIngester = new LogsIngester(new PointHandlerImpl("logs-ingester", pushValidationLevel, pushBlockedSamples, getFlushTasks("logs-ingester")), this::loadLogsIngestionConfig, prefix, System::currentTimeMillis);
logsIngester.start();
if (filebeatPort > 0) {
final Server filebeatServer = new Server(filebeatPort);
filebeatServer.setMessageListener(new FilebeatIngester(logsIngester, System::currentTimeMillis));
startAsManagedThread(() -> {
try {
activeListeners.inc();
filebeatServer.listen();
} catch (InterruptedException e) {
logger.log(Level.SEVERE, "Filebeat server interrupted.", e);
} catch (Exception e) {
// ChannelFuture throws undeclared checked exceptions, so we need to handle it
if (e instanceof BindException) {
logger.severe("Unable to start listener - port " + String.valueOf(rawLogsPort) + " is already in use!");
}
} finally {
activeListeners.dec();
}
}, "listener-logs-filebeat-" + filebeatPort);
}
if (rawLogsPort > 0) {
RawLogsIngester rawLogsIngester = new RawLogsIngester(logsIngester, rawLogsPort, System::currentTimeMillis);
startAsManagedThread(() -> {
try {
activeListeners.inc();
rawLogsIngester.listen();
} catch (InterruptedException e) {
logger.log(Level.SEVERE, "Raw logs server interrupted.", e);
} catch (Exception e) {
// ChannelFuture throws undeclared checked exceptions, so we need to handle it
if (e instanceof BindException) {
logger.severe("Unable to start listener - port " + String.valueOf(rawLogsPort) + " is already in use!");
}
} finally {
activeListeners.dec();
}
}, "listener-logs-raw-" + rawLogsPort);
}
} catch (ConfigurationException e) {
logger.log(Level.SEVERE, "Cannot start logsIngestion", e);
}
} else {
logger.info("Not loading logs ingestion -- no config specified.");
}
}
Aggregations