use of com.wavefront.agent.histogram.HistogramUtils.HistogramKeyMarshaller in project java by wavefrontHQ.
the class PushAgent method startHistogramListeners.
protected void startHistogramListeners(List<String> ports, ReportableEntityHandler<ReportPoint, String> pointHandler, SharedGraphiteHostAnnotator hostAnnotator, @Nullable Granularity granularity, int flushSecs, boolean memoryCacheEnabled, File baseDirectory, Long accumulatorSize, int avgKeyBytes, int avgDigestBytes, short compression, boolean persist, SpanSampler sampler) throws Exception {
if (ports.size() == 0)
return;
String listenerBinType = HistogramUtils.granularityToString(granularity);
// Accumulator
if (persist) {
// Check directory
checkArgument(baseDirectory.isDirectory(), baseDirectory.getAbsolutePath() + " must be a directory!");
checkArgument(baseDirectory.canWrite(), baseDirectory.getAbsolutePath() + " must be write-able!");
}
MapLoader<HistogramKey, AgentDigest, HistogramKeyMarshaller, AgentDigestMarshaller> mapLoader = new MapLoader<>(HistogramKey.class, AgentDigest.class, accumulatorSize, avgKeyBytes, avgDigestBytes, HistogramKeyMarshaller.get(), AgentDigestMarshaller.get(), persist);
File accumulationFile = new File(baseDirectory, "accumulator." + listenerBinType);
ChronicleMap<HistogramKey, AgentDigest> accumulator = mapLoader.get(accumulationFile);
histogramExecutor.scheduleWithFixedDelay(() -> {
// as ChronicleMap starts losing efficiency
if (accumulator.size() > accumulatorSize * 5) {
logger.severe("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + ") is more than 5x higher than currently configured size (" + accumulatorSize + "), which may cause severe performance degradation issues " + "or data loss! If the data volume is expected to stay at this level, we strongly " + "recommend increasing the value for accumulator size in wavefront.conf and " + "restarting the proxy.");
} else if (accumulator.size() > accumulatorSize * 2) {
logger.warning("Histogram " + listenerBinType + " accumulator size (" + accumulator.size() + ") is more than 2x higher than currently configured size (" + accumulatorSize + "), which may cause performance issues. If the data volume is " + "expected to stay at this level, we strongly recommend increasing the value " + "for accumulator size in wavefront.conf and restarting the proxy.");
}
}, 10, 10, TimeUnit.SECONDS);
AgentDigestFactory agentDigestFactory = new AgentDigestFactory(() -> (short) Math.min(compression, entityProps.getGlobalProperties().getHistogramStorageAccuracy()), TimeUnit.SECONDS.toMillis(flushSecs), proxyConfig.getTimeProvider());
Accumulator cachedAccumulator = new AccumulationCache(accumulator, agentDigestFactory, (memoryCacheEnabled ? accumulatorSize : 0), "histogram.accumulator." + HistogramUtils.granularityToString(granularity), null);
// Schedule write-backs
histogramExecutor.scheduleWithFixedDelay(cachedAccumulator::flush, proxyConfig.getHistogramAccumulatorResolveInterval(), proxyConfig.getHistogramAccumulatorResolveInterval(), TimeUnit.MILLISECONDS);
histogramFlushRunnables.add(cachedAccumulator::flush);
PointHandlerDispatcher dispatcher = new PointHandlerDispatcher(cachedAccumulator, pointHandler, proxyConfig.getTimeProvider(), () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), proxyConfig.getHistogramAccumulatorFlushMaxBatchSize() < 0 ? null : proxyConfig.getHistogramAccumulatorFlushMaxBatchSize(), granularity);
histogramExecutor.scheduleWithFixedDelay(dispatcher, proxyConfig.getHistogramAccumulatorFlushInterval(), proxyConfig.getHistogramAccumulatorFlushInterval(), TimeUnit.MILLISECONDS);
histogramFlushRunnables.add(dispatcher);
// gracefully shutdown persisted accumulator (ChronicleMap) on proxy exit
shutdownTasks.add(() -> {
try {
logger.fine("Flushing in-flight histogram accumulator digests: " + listenerBinType);
cachedAccumulator.flush();
logger.fine("Shutting down histogram accumulator cache: " + listenerBinType);
accumulator.close();
} catch (Throwable t) {
logger.log(Level.SEVERE, "Error flushing " + listenerBinType + " accumulator, possibly unclean shutdown: ", t);
}
});
ReportableEntityHandlerFactory histogramHandlerFactory = new ReportableEntityHandlerFactory() {
private final Map<HandlerKey, ReportableEntityHandler<?, ?>> handlers = new ConcurrentHashMap<>();
@SuppressWarnings("unchecked")
@Override
public <T, U> ReportableEntityHandler<T, U> getHandler(HandlerKey handlerKey) {
return (ReportableEntityHandler<T, U>) handlers.computeIfAbsent(handlerKey, k -> new HistogramAccumulationHandlerImpl(handlerKey, cachedAccumulator, proxyConfig.getPushBlockedSamples(), granularity, validationConfiguration, granularity == null, null, blockedHistogramsLogger, VALID_HISTOGRAMS_LOGGER));
}
@Override
public void shutdown(@Nonnull String handle) {
handlers.values().forEach(ReportableEntityHandler::shutdown);
}
};
ports.forEach(strPort -> {
int port = Integer.parseInt(strPort);
registerPrefixFilter(strPort);
registerTimestampFilter(strPort);
if (proxyConfig.isHttpHealthCheckAllPorts()) {
healthCheckManager.enableHealthcheck(port);
}
WavefrontPortUnificationHandler wavefrontPortUnificationHandler = new WavefrontPortUnificationHandler(strPort, tokenAuthenticator, healthCheckManager, decoderSupplier.get(), histogramHandlerFactory, hostAnnotator, preprocessors.get(strPort), () -> entityProps.get(ReportableEntityType.HISTOGRAM).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE).isFeatureDisabled(), () -> entityProps.get(ReportableEntityType.TRACE_SPAN_LOGS).isFeatureDisabled(), sampler);
startAsManagedThread(port, new TcpIngester(createInitializer(wavefrontPortUnificationHandler, port, proxyConfig.getHistogramMaxReceivedLength(), proxyConfig.getHistogramHttpBufferSize(), proxyConfig.getListenerIdleConnectionTimeout(), getSslContext(strPort), getCorsConfig(strPort)), port).withChildChannelOptions(childChannelOptions), "listener-histogram-" + port);
logger.info("listening on port: " + port + " for histogram samples, accumulating to the " + listenerBinType);
});
}
Aggregations